Skip to content

L
linux
Commit e4b9e2aa authored by Andi Kleen's avatar Andi Kleen Committed by Linus Torvalds
Browse files
Options

[PATCH] 2.5.21 x86-64 jumbo patch - arch specific changes 

Here is the big 2.5.21 x86-64 sync patch. It only touches arch/x86_64
and include/asm-x86_64. It requires a few other changes that I'm sending
in separate mail.

Changes:
- merge wit 2.5.21
- merge from 2.5.21/i386 (new PCI code, new LDT code etc.)
- sync with 2.4-x86_64 tree.
- minor updates to 32bit emulation
- better early console; including serial support.
- now set up dummy PDA for booting to avoid problems
- Fix GS reloading in context switch one instruction race
- Remove hardcoded names from mpparse code
- Fix inline assembly for RAID-5 xor (similar change needed for i386)
- Real per cpu data support based on PDA field
- Cleanup of offset.c generation requested by Kai: it only puts structure
  offsets into offset.h now.
- Fix i387 fxsave signal frame problems.
- Add uname emulation via personality ("linux32")
- New SSE optimized checksum-copy, copy*user, memcpy, clear_page, copy_page
  functions. Other tunings/cleanups in checksum and other user memory
  access function.
- check if exception table is really sorted
- Cleanups in page table handling in preparation of non executable pages
  support.
- Cleanup PDA access to not require offset.h (thanks to kai for kicking me
  to this)
- use long long for u64/s64 to avoid more warnings
- remove CONFIG_ISA
- fix various bugs and other cleanups
parent cc9af0c5
Hide whitespace changes
Inline Side-by-side
Showing with 3023 additions and 2289 deletions
arch/x86_64/Makefile
View file @ e4b9e2aa
......@@ -23,7 +23,7 @@
#
# early bootup linking needs 32bit. You can either use real 32bit tools
# here or 64bit tools switch to 32bit mode.
# here or 64bit tools in 32bit mode.
#
IA32_CC := $(CROSS_COMPILE)gcc -m32 -O2 -fomit-frame-pointer -nostdinc -I $(HPATH)
IA32_LD := $(CROSS_COMPILE)ld -m elf_i386
......@@ -41,11 +41,12 @@ LINKFLAGS =-T $(TOPDIR)/arch/x86_64/vmlinux.lds $(LDFLAGS)
CFLAGS += -mno-red-zone
CFLAGS += -mcmodel=kernel
CFLAGS += -pipe
# this makes reading assembly source easier
CFLAGS += -fno-reorder-blocks
# needed for later gcc 3.1
CFLAGS += -finline-limit=2000
# needed for earlier gcc 3.1
CFLAGS += -fno-strength-reduce
#CFLAGS += -fno-strength-reduce
#CFLAGS += -g
# prevent gcc from keeping the stack 16 byte aligned (FIXME)
......@@ -63,9 +64,9 @@ SUBDIRS += arch/x86_64/ia32
CORE_FILES += arch/x86_64/ia32/ia32.o
endif
ifdef CONFIG_HOSTFS
SUBDIRS += arch/x86_64/hostfs
core-$(CONFIG_HOSTFS) += arch/x86_64/hostfs/hostfs.o
ifdef CONFIG_PCI
SUBDIRS += arch/x86_64/pci
DRIVERS += arch/x86_64/pci/pci.o
endif
CORE_FILES += $(core-y)
......@@ -77,7 +78,7 @@ vmlinux: arch/x86_64/vmlinux.lds
.PHONY: zImage bzImage compressed zlilo bzlilo zdisk bzdisk install \
clean archclean archmrproper archdep checkoffset
checkoffset: FORCE
checkoffset: FORCE include/asm
make -C arch/$(ARCH)/tools $(TOPDIR)/include/asm-x86_64/offset.h
bzImage: checkoffset vmlinux
......
arch/x86_64/boot/setup.S
View file @ e4b9e2aa
......@@ -452,7 +452,7 @@ no_psmouse:
cmpw $0, %cs:realmode_swtch
jz rmodeswtch_normal
lcall %cs:realmode_swtch
lcall *%cs:realmode_swtch
jmp rmodeswtch_end
......
arch/x86_64/boot/video.S
View file @ e4b9e2aa
......@@ -437,6 +437,7 @@ setalias:
# Setting of user mode (AX=mode ID) => CF=success
mode_set:
movw %ax, %fs:(0x01fa)
movw %ax, %bx
cmpb $0xff, %ah
jz setalias
......
arch/x86_64/config.in
View file @ e4b9e2aa
......@@ -2,12 +2,17 @@
# For a description of the syntax of this configuration file,
# see Documentation/kbuild/config-language.txt.
#
# Note: ISA is disabled and will hopefully never be enabled.
# If you managed to buy an ISA x86-64 box you'll have to fix all the
# ISA drivers you need yourself.
#
mainmenu_name "Linux Kernel Configuration"
define_bool CONFIG_X86_64 y
define_bool CONFIG_X86 y
define_bool CONFIG_ISA y
define_bool CONFIG_ISA n
define_bool CONFIG_SBUS n
define_bool CONFIG_UID16 y
......@@ -22,7 +27,8 @@ source init/Config.in
mainmenu_option next_comment
comment 'Processor type and features'
choice 'Processor family' \
"AMD-Hammer CONFIG_MK8" CONFIG_MK8
"AMD-Hammer CONFIG_MK8 \
Generic-x86-64 CONFIG_GENERIC_CPU" AMD-Hammer
#
# Define implied options from the CPU selection here
......@@ -44,8 +50,10 @@ define_bool CONFIG_X86_LOCAL_APIC y
#currently broken:
#bool 'MTRR (Memory Type Range Register) support' CONFIG_MTRR
bool 'Symmetric multi-processing support' CONFIG_SMP
bool 'Preemptible Kernel' CONFIG_PREEMPT
if [ "$CONFIG_SMP" = "y" -a "$CONFIG_X86_CMPXCHG" = "y" ]; then
if [ "$CONFIG_SMP" = "n" ]; then
bool 'Preemptible Kernel' CONFIG_PREEMPT
fi
if [ "$CONFIG_SMP" = "y" ]; then
define_bool CONFIG_HAVE_DEC_LOCK y
fi
......@@ -56,10 +64,18 @@ endmenu
mainmenu_option next_comment
comment 'General options'
comment 'Power management options'
bool 'Power Management support' CONFIG_PM
source drivers/acpi/Config.in
endmenu
mainmenu_option next_comment
comment 'Bus options (PCI etc.)'
bool 'PCI support' CONFIG_PCI
if [ "$CONFIG_PCI" = "y" ]; then
# x86-64 doesn't support PCI BIOS access from long mode so always go direct.
......@@ -77,6 +93,12 @@ else
define_bool CONFIG_PCMCIA n
fi
endmenu
mainmenu_option next_comment
comment 'Executable file formats / Emulations'
if [ "$CONFIG_PROC_FS" = "y" ]; then
define_bool CONFIG_KCORE_ELF y
fi
......@@ -84,8 +106,6 @@ fi
tristate 'Kernel support for ELF binaries' CONFIG_BINFMT_ELF
tristate 'Kernel support for MISC binaries' CONFIG_BINFMT_MISC
bool 'Power Management support' CONFIG_PM
bool 'IA32 Emulation' CONFIG_IA32_EMULATION
endmenu
......@@ -94,7 +114,7 @@ source drivers/mtd/Config.in
source drivers/parport/Config.in
source drivers/pnp/Config.in
#source drivers/pnp/Config.in
source drivers/block/Config.in
......@@ -142,9 +162,10 @@ if [ "$CONFIG_NET" = "y" ]; then
bool 'Network device support' CONFIG_NETDEVICES
if [ "$CONFIG_NETDEVICES" = "y" ]; then
source drivers/net/Config.in
if [ "$CONFIG_ATM" = "y" ]; then
source drivers/atm/Config.in
fi
# ATM seems to be largely 64bit unsafe and also unmaintained - disable it for now.
# if [ "$CONFIG_ATM" = "y" ]; then
# source drivers/atm/Config.in
# fi
fi
endmenu
fi
......@@ -155,14 +176,7 @@ source net/irda/Config.in
source drivers/isdn/Config.in
mainmenu_option next_comment
comment 'Old CD-ROM drivers (not SCSI, not IDE)'
bool 'Support non-SCSI/IDE/ATAPI CDROM drives' CONFIG_CD_NO_IDESCSI
if [ "$CONFIG_CD_NO_IDESCSI" != "n" ]; then
source drivers/cdrom/Config.in
fi
endmenu
# no support for non IDE/SCSI cdroms as they were all ISA only
#
# input before char - char/joystick depends on it. As does USB.
......
arch/x86_64/defconfig
View file @ e4b9e2aa
#
# Automatically generated make config: don't edit
# Automatically generated by make menuconfig: don't edit
#
CONFIG_X86_64=y
CONFIG_X86=y
CONFIG_ISA=y
# CONFIG_ISA is not set
# CONFIG_SBUS is not set
CONFIG_UID16=y
CONFIG_RWSEM_GENERIC_SPINLOCK=y
......@@ -35,6 +35,7 @@ CONFIG_MODULES=y
# Processor type and features
#
CONFIG_MK8=y
# CONFIG_GENERIC_CPU is not set
CONFIG_X86_L1_CACHE_BYTES=64
CONFIG_X86_L1_CACHE_SHIFT=6
CONFIG_X86_TSC=y
......@@ -47,28 +48,35 @@ CONFIG_X86_CPUID=y
CONFIG_X86_IO_APIC=y
CONFIG_X86_LOCAL_APIC=y
CONFIG_SMP=y
# CONFIG_PREEMPT is not set
CONFIG_HAVE_DEC_LOCK=y
CONFIG_X86_MCE=y
# CONFIG_X86_MCE_NONFATAL is not set
#
# General options
# Power management options
#
# CONFIG_PM is not set
#
# ACPI Support
#
# CONFIG_ACPI is not set
#
# Bus options (PCI etc.)
#
CONFIG_PCI=y
CONFIG_PCI_DIRECT=y
# CONFIG_PCI_NAMES is not set
# CONFIG_HOTPLUG is not set
# CONFIG_PCMCIA is not set
#
# Executable file formats / Emulations
#
CONFIG_KCORE_ELF=y
CONFIG_BINFMT_ELF=y
# CONFIG_BINFMT_MISC is not set
CONFIG_PM=y
CONFIG_IA32_EMULATION=y
#
......@@ -81,27 +89,22 @@ CONFIG_IA32_EMULATION=y
#
# CONFIG_PARPORT is not set
#
# Plug and Play configuration
#
# CONFIG_PNP is not set
# CONFIG_ISAPNP is not set
# CONFIG_PNPBIOS is not set
#
# Block devices
#
# CONFIG_BLK_DEV_FD is not set
CONFIG_BLK_DEV_FD=y
# CONFIG_BLK_DEV_XD is not set
# CONFIG_PARIDE is not set
# CONFIG_BLK_CPQ_DA is not set
# CONFIG_BLK_CPQ_CISS_DA is not set
# CONFIG_CISS_SCSI_TAPE is not set
# CONFIG_BLK_DEV_DAC960 is not set
# CONFIG_BLK_DEV_LOOP is not set
# CONFIG_BLK_DEV_UMEM is not set
CONFIG_BLK_DEV_LOOP=y
# CONFIG_BLK_DEV_NBD is not set
# CONFIG_BLK_DEV_RAM is not set
# CONFIG_BLK_DEV_INITRD is not set
CONFIG_BLK_DEV_RAM=y
CONFIG_BLK_DEV_RAM_SIZE=4096
CONFIG_BLK_DEV_INITRD=y
#
# Multi-device support (RAID and LVM)
......@@ -118,17 +121,19 @@ CONFIG_IA32_EMULATION=y
#
# Networking options
#
# CONFIG_PACKET is not set
# CONFIG_NETLINK_DEV is not set
CONFIG_PACKET=y
# CONFIG_PACKET_MMAP is not set
CONFIG_NETLINK_DEV=y
# CONFIG_NETFILTER is not set
# CONFIG_FILTER is not set
CONFIG_FILTER=y
CONFIG_UNIX=y
CONFIG_INET=y
# CONFIG_IP_MULTICAST is not set
CONFIG_IP_MULTICAST=y
# CONFIG_IP_ADVANCED_ROUTER is not set
# CONFIG_IP_PNP is not set
# CONFIG_NET_IPIP is not set
# CONFIG_NET_IPGRE is not set
# CONFIG_IP_MROUTE is not set
# CONFIG_ARPD is not set
# CONFIG_INET_ECN is not set
# CONFIG_SYN_COOKIES is not set
......@@ -136,12 +141,13 @@ CONFIG_INET=y
# CONFIG_KHTTPD is not set
# CONFIG_ATM is not set
# CONFIG_VLAN_8021Q is not set
# CONFIG_IPX is not set
# CONFIG_ATALK is not set
#
# Appletalk devices
#
#
# CONFIG_IPX is not set
# CONFIG_ATALK is not set
# CONFIG_DEV_APPLETALK is not set
# CONFIG_DECNET is not set
# CONFIG_BRIDGE is not set
# CONFIG_X25 is not set
......@@ -174,10 +180,6 @@ CONFIG_IDE=y
# ATA and ATAPI Block devices
#
CONFIG_BLK_DEV_IDE=y
#
# Please see Documentation/ide.txt for help/info on IDE drives
#
# CONFIG_BLK_DEV_HD_IDE is not set
# CONFIG_BLK_DEV_HD is not set
CONFIG_BLK_DEV_IDEDISK=y
......@@ -188,10 +190,6 @@ CONFIG_BLK_DEV_IDECD=y
# CONFIG_BLK_DEV_IDETAPE is not set
# CONFIG_BLK_DEV_IDEFLOPPY is not set
# CONFIG_BLK_DEV_IDESCSI is not set
#
# IDE chipset support
#
# CONFIG_BLK_DEV_CMD640 is not set
# CONFIG_BLK_DEV_CMD640_ENHANCED is not set
# CONFIG_BLK_DEV_ISAPNP is not set
......@@ -199,17 +197,17 @@ CONFIG_BLK_DEV_IDECD=y
# CONFIG_BLK_DEV_OFFBOARD is not set
# CONFIG_IDEPCI_SHARE_IRQ is not set
CONFIG_BLK_DEV_IDEDMA_PCI=y
CONFIG_IDEDMA_PCI_AUTO=y
# CONFIG_IDEDMA_PCI_AUTO is not set
# CONFIG_IDEDMA_ONLYDISK is not set
CONFIG_BLK_DEV_IDEDMA=y
# CONFIG_BLK_DEV_IDE_TCQ is not set
# CONFIG_BLK_DEV_IDE_TCQ_DEFAULT is not set
# CONFIG_IDEDMA_NEW_DRIVE_LISTINGS is not set
# CONFIG_BLK_DEV_AEC62XX is not set
# CONFIG_AEC62XX_TUNING is not set
# CONFIG_AEC6280_BURST is not set
# CONFIG_BLK_DEV_ALI15X3 is not set
# CONFIG_WDC_ALI15X3 is not set
# CONFIG_BLK_DEV_AMD74XX is not set
CONFIG_BLK_DEV_AMD74XX=y
# CONFIG_BLK_DEV_CMD64X is not set
# CONFIG_BLK_DEV_CY82C693 is not set
# CONFIG_BLK_DEV_CS5530 is not set
......@@ -226,9 +224,11 @@ CONFIG_BLK_DEV_IDEDMA=y
# CONFIG_BLK_DEV_SIS5513 is not set
# CONFIG_BLK_DEV_TRM290 is not set
# CONFIG_BLK_DEV_VIA82CXXX is not set
# CONFIG_BLK_DEV_SL82C105 is not set
# CONFIG_IDE_CHIPSETS is not set
# CONFIG_IDEDMA_IVB is not set
CONFIG_IDEDMA_AUTO=y
CONFIG_ATAPI=y
# CONFIG_IDEDMA_AUTO is not set
# CONFIG_BLK_DEV_ATARAID is not set
# CONFIG_BLK_DEV_ATARAID_PDC is not set
# CONFIG_BLK_DEV_ATARAID_HPT is not set
......@@ -236,16 +236,67 @@ CONFIG_IDEDMA_AUTO=y
#
# SCSI support
#
# CONFIG_SCSI is not set
CONFIG_SCSI=y
CONFIG_BLK_DEV_SD=y
CONFIG_SD_EXTRA_DEVS=40
# CONFIG_CHR_DEV_ST is not set
# CONFIG_CHR_DEV_OSST is not set
# CONFIG_BLK_DEV_SR is not set
# CONFIG_CHR_DEV_SG is not set
# CONFIG_SCSI_MULTI_LUN is not set
# CONFIG_SCSI_REPORT_LUNS is not set
CONFIG_SCSI_CONSTANTS=y
# CONFIG_SCSI_LOGGING is not set
#
# SCSI low-level drivers
#
# CONFIG_BLK_DEV_3W_XXXX_RAID is not set
# CONFIG_SCSI_7000FASST is not set
# CONFIG_SCSI_ACARD is not set
CONFIG_SCSI_AIC7XXX=y
CONFIG_AIC7XXX_CMDS_PER_DEVICE=253
CONFIG_AIC7XXX_RESET_DELAY_MS=15000
# CONFIG_AIC7XXX_BUILD_FIRMWARE is not set
# CONFIG_SCSI_ADVANSYS is not set
# CONFIG_SCSI_IN2000 is not set
# CONFIG_SCSI_MEGARAID is not set
# CONFIG_SCSI_BUSLOGIC is not set
# CONFIG_SCSI_CPQFCTS is not set
# CONFIG_SCSI_DMX3191D is not set
# CONFIG_SCSI_DTC3280 is not set
# CONFIG_SCSI_EATA is not set
# CONFIG_SCSI_EATA_DMA is not set
# CONFIG_SCSI_EATA_PIO is not set
# CONFIG_SCSI_FUTURE_DOMAIN is not set
# CONFIG_SCSI_GDTH is not set
# CONFIG_SCSI_GENERIC_NCR5380 is not set
# CONFIG_SCSI_IPS is not set
# CONFIG_SCSI_INITIO is not set
# CONFIG_SCSI_INIA100 is not set
# CONFIG_SCSI_NCR53C406A is not set
# CONFIG_SCSI_NCR53C7xx is not set
# CONFIG_SCSI_SYM53C8XX_2 is not set
# CONFIG_SCSI_NCR53C8XX is not set
# CONFIG_SCSI_SYM53C8XX is not set
# CONFIG_SCSI_PAS16 is not set
# CONFIG_SCSI_PCI2000 is not set
# CONFIG_SCSI_PCI2220I is not set
# CONFIG_SCSI_PSI240I is not set
# CONFIG_SCSI_QLOGIC_FAS is not set
# CONFIG_SCSI_QLOGIC_ISP is not set
# CONFIG_SCSI_QLOGIC_FC is not set
# CONFIG_SCSI_QLOGIC_1280 is not set
# CONFIG_SCSI_SYM53C416 is not set
# CONFIG_SCSI_DC390T is not set
# CONFIG_SCSI_T128 is not set
# CONFIG_SCSI_U14_34F is not set
# CONFIG_SCSI_DEBUG is not set
#
# Fusion MPT device support
#
# CONFIG_FUSION is not set
# CONFIG_FUSION_BOOT is not set
# CONFIG_FUSION_ISENSE is not set
# CONFIG_FUSION_CTL is not set
# CONFIG_FUSION_LAN is not set
#
# IEEE 1394 (FireWire) support (EXPERIMENTAL)
......@@ -255,7 +306,83 @@ CONFIG_IDEDMA_AUTO=y
#
# Network device support
#
# CONFIG_NETDEVICES is not set
CONFIG_NETDEVICES=y
#
# ARCnet devices
#
# CONFIG_ARCNET is not set
# CONFIG_DUMMY is not set
# CONFIG_BONDING is not set
# CONFIG_EQUALIZER is not set
# CONFIG_TUN is not set
# CONFIG_ETHERTAP is not set
#
# Ethernet (10 or 100Mbit)
#
CONFIG_NET_ETHERNET=y
# CONFIG_SUNLANCE is not set
# CONFIG_HAPPYMEAL is not set
# CONFIG_SUNBMAC is not set
# CONFIG_SUNQE is not set
# CONFIG_SUNGEM is not set
CONFIG_NET_VENDOR_3COM=y
# CONFIG_EL1 is not set
# CONFIG_EL2 is not set
# CONFIG_ELPLUS is not set
# CONFIG_EL16 is not set
# CONFIG_ELMC is not set
# CONFIG_ELMC_II is not set
CONFIG_VORTEX=y
# CONFIG_LANCE is not set
# CONFIG_NET_VENDOR_SMC is not set
# CONFIG_NET_VENDOR_RACAL is not set
# CONFIG_HP100 is not set
# CONFIG_NET_ISA is not set
# CONFIG_NET_PCI is not set
# CONFIG_NET_POCKET is not set
#
# Ethernet (1000 Mbit)
#
# CONFIG_ACENIC is not set
# CONFIG_DL2K is not set
# CONFIG_E1000 is not set
# CONFIG_MYRI_SBUS is not set
# CONFIG_NS83820 is not set
# CONFIG_HAMACHI is not set
# CONFIG_YELLOWFIN is not set
# CONFIG_SK98LIN is not set
CONFIG_TIGON3=y
# CONFIG_FDDI is not set
# CONFIG_HIPPI is not set
# CONFIG_PLIP is not set
# CONFIG_PPP is not set
# CONFIG_SLIP is not set
#
# Wireless LAN (non-hamradio)
#
# CONFIG_NET_RADIO is not set
#
# Token Ring devices
#
# CONFIG_TR is not set
# CONFIG_NET_FC is not set
# CONFIG_RCPCI is not set
# CONFIG_SHAPER is not set
#
# Wan interfaces
#
# CONFIG_WAN is not set
#
# Tulip family network device support
#
# CONFIG_NET_TULIP is not set
#
# Amateur Radio support
......@@ -272,11 +399,6 @@ CONFIG_IDEDMA_AUTO=y
#
# CONFIG_ISDN_BOOL is not set
#
# Old CD-ROM drivers (not SCSI, not IDE)
#
# CONFIG_CD_NO_IDESCSI is not set
#
# Input device support
#
......@@ -329,7 +451,7 @@ CONFIG_PSMOUSE=y
# CONFIG_WATCHDOG is not set
# CONFIG_INTEL_RNG is not set
# CONFIG_NVRAM is not set
# CONFIG_RTC is not set
CONFIG_RTC=y
# CONFIG_DTLK is not set
# CONFIG_R3964 is not set
# CONFIG_APPLICOM is not set
......@@ -356,9 +478,11 @@ CONFIG_PSMOUSE=y
# File systems
#
# CONFIG_QUOTA is not set
# CONFIG_AUTOFS_FS is not set
# CONFIG_QFMT_V1 is not set
# CONFIG_QFMT_V2 is not set
CONFIG_AUTOFS_FS=y
# CONFIG_AUTOFS4_FS is not set
# CONFIG_REISERFS_FS is not set
CONFIG_REISERFS_FS=y
# CONFIG_REISERFS_CHECK is not set
# CONFIG_REISERFS_PROC_INFO is not set
# CONFIG_ADFS_FS is not set
......@@ -366,8 +490,8 @@ CONFIG_PSMOUSE=y
# CONFIG_AFFS_FS is not set
# CONFIG_HFS_FS is not set
# CONFIG_BFS_FS is not set
# CONFIG_EXT3_FS is not set
# CONFIG_JBD is not set
CONFIG_EXT3_FS=y
CONFIG_JBD=y
# CONFIG_JBD_DEBUG is not set
# CONFIG_FAT_FS is not set
# CONFIG_MSDOS_FS is not set
......@@ -377,7 +501,7 @@ CONFIG_PSMOUSE=y
# CONFIG_JFFS_FS is not set
# CONFIG_JFFS2_FS is not set
# CONFIG_CRAMFS is not set
# CONFIG_TMPFS is not set
CONFIG_TMPFS=y
CONFIG_RAMFS=y
# CONFIG_ISO9660_FS is not set
# CONFIG_JOLIET is not set
......@@ -410,15 +534,15 @@ CONFIG_EXT2_FS=y
#
# CONFIG_CODA_FS is not set
# CONFIG_INTERMEZZO_FS is not set
# CONFIG_NFS_FS is not set
CONFIG_NFS_FS=y
# CONFIG_NFS_V3 is not set
# CONFIG_ROOT_NFS is not set
# CONFIG_NFSD is not set
CONFIG_NFSD=y
# CONFIG_NFSD_V3 is not set
# CONFIG_NFSD_TCP is not set
# CONFIG_SUNRPC is not set
# CONFIG_LOCKD is not set
# CONFIG_EXPORTFS is not set
CONFIG_SUNRPC=y
CONFIG_LOCKD=y
CONFIG_EXPORTFS=y
# CONFIG_SMB_FS is not set
# CONFIG_NCP_FS is not set
# CONFIG_NCPFS_PACKET_SIGNING is not set
......@@ -471,7 +595,7 @@ CONFIG_VGA_CONSOLE=y
#
CONFIG_DEBUG_KERNEL=y
# CONFIG_DEBUG_SLAB is not set
# CONFIG_MAGIC_SYSRQ is not set
CONFIG_MAGIC_SYSRQ=y
# CONFIG_DEBUG_SPINLOCK is not set
# CONFIG_CHECKING is not set
# CONFIG_INIT_DEBUG is not set
......
arch/x86_64/ia32/Makefile
View file @ e4b9e2aa
......@@ -2,11 +2,16 @@
# Makefile for the ia32 kernel emulation subsystem.
#
O_TARGET := ia32.o
USE_STANDARD_AS_RULE := true
export-objs := ia32_ioctl.o sys_ia32.o
export-objs := ia32_ioctl.o
all: ia32.o
O_TARGET := ia32.o
obj-$(CONFIG_IA32_EMULATION) := ia32entry.o sys_ia32.o ia32_ioctl.o ia32_signal.o \
ia32_binfmt.o fpu32.o socket32.o ptrace32.o
clean::
include $(TOPDIR)/Rules.make
arch/x86_64/ia32/ia32_ioctl.c
View file @ e4b9e2aa
......@@ -113,6 +113,7 @@ static int w_long(unsigned int fd, unsigned int cmd, unsigned long arg)
return err;
}
#if 0
static int rw_long(unsigned int fd, unsigned int cmd, unsigned long arg)
{
mm_segment_t old_fs = get_fs();
......@@ -128,6 +129,7 @@ static int rw_long(unsigned int fd, unsigned int cmd, unsigned long arg)
return -EFAULT;
return err;
}
#endif
static int do_ext2_ioctl(unsigned int fd, unsigned int cmd, unsigned long arg)
{
......@@ -2971,11 +2973,6 @@ static int blkpg_ioctl_trans(unsigned int fd, unsigned int cmd, struct blkpg_ioc
return err;
}
static int ioc_settimeout(unsigned int fd, unsigned int cmd, unsigned long arg)
{
return rw_long(fd, AUTOFS_IOC_SETTIMEOUT, arg);
}
/* SuSE extension */
#ifndef TIOCGDEV
#define TIOCGDEV _IOR('T',0x32, unsigned int)
......@@ -3087,19 +3084,13 @@ static int serial_struct_ioctl(unsigned fd, unsigned cmd, void *ptr)
return err;
}
struct ioctl_trans {
unsigned long cmd;
int (*handler)(unsigned int, unsigned int, unsigned long, struct file * filp);
struct ioctl_trans *next;
};
/* generic function to change a single long put_user to arg to 32bit */
static int arg2long(unsigned int fd, unsigned int cmd, unsigned long arg)
static int generic_long_put(unsigned int fd, unsigned int cmd, unsigned long arg)
{
int ret;
unsigned long val = 0;
mm_segment_t oldseg = get_fs();
set_fs(KERNEL_DS);
cmd = (cmd & 0xc000ffff) | (sizeof(unsigned long) << _IOC_SIZESHIFT);
ret = sys_ioctl(fd, cmd, (unsigned long)&val);
set_fs(oldseg);
if (!ret || val) {
......@@ -3109,6 +3100,29 @@ static int arg2long(unsigned int fd, unsigned int cmd, unsigned long arg)
return ret;
}
static int generic_long_get(unsigned int fd, unsigned int cmd, unsigned long arg)
{
int ret;
unsigned int ival;
unsigned long val = 0;
mm_segment_t oldseg = get_fs();
if (get_user(ival, (unsigned int *)arg))
return -EFAULT;
val = ival;
set_fs(KERNEL_DS);
cmd = (cmd & 0xc000ffff) | (sizeof(unsigned long) << _IOC_SIZESHIFT);
ret = sys_ioctl(fd, cmd, (unsigned long)&val);
set_fs(oldseg);
return ret;
}
struct ioctl_trans {
unsigned long cmd;
int (*handler)(unsigned int, unsigned int, unsigned long, struct file * filp);
struct ioctl_trans *next;
};
#define REF_SYMBOL(handler) if (0) (void)handler;
#define HANDLE_IOCTL2(cmd,handler) REF_SYMBOL(handler); asm volatile(".quad %c0, " #handler ",0"::"i" (cmd));
#define HANDLE_IOCTL(cmd,handler) HANDLE_IOCTL2(cmd,handler)
......@@ -3316,10 +3330,6 @@ COMPATIBLE_IOCTL(_IOR('v' , BASE_VIDIOCPRIVATE+5, int))
COMPATIBLE_IOCTL(_IOR('v' , BASE_VIDIOCPRIVATE+6, int))
COMPATIBLE_IOCTL(_IOR('v' , BASE_VIDIOCPRIVATE+7, int))
/* Little p (/dev/rtc, /dev/envctrl, etc.) */
#if 0
COMPATIBLE_IOCTL(_IOR('p', 20, int[7])) /* RTCGET */
COMPATIBLE_IOCTL(_IOW('p', 21, int[7])) /* RTCSET */
#endif
COMPATIBLE_IOCTL(RTC_AIE_ON)
COMPATIBLE_IOCTL(RTC_AIE_OFF)
COMPATIBLE_IOCTL(RTC_UIE_ON)
......@@ -3334,10 +3344,14 @@ COMPATIBLE_IOCTL(RTC_RD_TIME)
COMPATIBLE_IOCTL(RTC_SET_TIME)
COMPATIBLE_IOCTL(RTC_WKALM_SET)
COMPATIBLE_IOCTL(RTC_WKALM_RD)
HANDLE_IOCTL(RTC_IRQP_READ,arg2long)
COMPATIBLE_IOCTL(RTC_IRQP_SET)
COMPATIBLE_IOCTL(RTC_EPOCH_READ)
COMPATIBLE_IOCTL(RTC_EPOCH_SET)
#define RTC_IRQP_READ32 _IOR('p', 0x0b, unsigned int) /* Read IRQ rate */
HANDLE_IOCTL(RTC_IRQP_READ32,generic_long_put)
#define RTC_IRQP_SET32 _IOW('p', 0x0c, unsigned int) /* Set IRQ rate */
HANDLE_IOCTL(RTC_IRQP_SET32,generic_long_get)
#define RTC_EPOCH_READ32 _IOR('p', 0x0d, unsigned long) /* Read epoch */
#define RTC_EPOCH_SET32 _IOW('p', 0x0e, unsigned long) /* Set epoch */
HANDLE_IOCTL(RTC_EPOCH_READ32, generic_long_put)
HANDLE_IOCTL(RTC_EPOCH_SET32, generic_long_get)
/* Little m */
COMPATIBLE_IOCTL(MTIOCTOP)
/* Socket level stuff */
......@@ -3605,6 +3619,8 @@ COMPATIBLE_IOCTL(AUTOFS_IOC_FAIL)
COMPATIBLE_IOCTL(AUTOFS_IOC_CATATONIC)
COMPATIBLE_IOCTL(AUTOFS_IOC_PROTOVER)
COMPATIBLE_IOCTL(AUTOFS_IOC_EXPIRE)
#define AUTOFS_IOC_SETTIMEOUT32 _IOWR(0x93,0x64,unsigned int)
HANDLE_IOCTL(AUTOFS_IOC_SETTIMEOUT32, generic_long_get);
/* DEVFS */
COMPATIBLE_IOCTL(DEVFSDIOC_GET_PROTO_REV)
COMPATIBLE_IOCTL(DEVFSDIOC_SET_EVENT_MASK)
......@@ -3671,14 +3687,6 @@ COMPATIBLE_IOCTL(DRM_IOCTL_LOCK)
COMPATIBLE_IOCTL(DRM_IOCTL_UNLOCK)
COMPATIBLE_IOCTL(DRM_IOCTL_FINISH)
#endif /* DRM */
#ifdef CONFIG_AUTOFS_FS
COMPATIBLE_IOCTL(AUTOFS_IOC_READY);
COMPATIBLE_IOCTL(AUTOFS_IOC_FAIL);
COMPATIBLE_IOCTL(AUTOFS_IOC_CATATONIC);
COMPATIBLE_IOCTL(AUTOFS_IOC_PROTOVER);
COMPATIBLE_IOCTL(AUTOFS_IOC_SETTIMEOUT);
COMPATIBLE_IOCTL(AUTOFS_IOC_EXPIRE);
#endif
COMPATIBLE_IOCTL(REISERFS_IOC_UNPACK);
/* serial driver */
HANDLE_IOCTL(TIOCGSERIAL, serial_struct_ioctl);
......@@ -3771,8 +3779,6 @@ HANDLE_IOCTL(CDROMREADALL, cdrom_ioctl_trans)
HANDLE_IOCTL(CDROM_SEND_PACKET, cdrom_ioctl_trans)
HANDLE_IOCTL(LOOP_SET_STATUS, loop_status)
HANDLE_IOCTL(LOOP_GET_STATUS, loop_status)
#define AUTOFS_IOC_SETTIMEOUT32 _IOWR(0x93,0x64,unsigned int)
HANDLE_IOCTL(AUTOFS_IOC_SETTIMEOUT32, ioc_settimeout)
HANDLE_IOCTL(PIO_FONTX, do_fontx_ioctl)
HANDLE_IOCTL(GIO_FONTX, do_fontx_ioctl)
HANDLE_IOCTL(PIO_UNIMAP, do_unimap_ioctl)
......
arch/x86_64/ia32/ia32_signal.c
View file @ e4b9e2aa
......@@ -241,7 +241,7 @@ asmlinkage int sys32_sigreturn(struct pt_regs regs)
return eax;
badframe:
force_sig(SIGSEGV, current);
signal_fault(&regs, frame, "32bit sigreturn");
return 0;
}
......@@ -280,7 +280,7 @@ asmlinkage int sys32_rt_sigreturn(struct pt_regs regs)
return eax;
badframe:
force_sig(SIGSEGV, current);
signal_fault(&regs, frame, "32bit rt sigreturn");
return 0;
}
......@@ -420,7 +420,7 @@ void ia32_setup_frame(int sig, struct k_sigaction *ka,
give_sigsegv:
if (sig == SIGSEGV)
ka->sa.sa_handler = SIG_DFL;
force_sig(SIGSEGV, current);
signal_fault(regs,frame,"32bit signal setup");
}
void ia32_setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
......@@ -493,6 +493,6 @@ void ia32_setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
give_sigsegv:
if (sig == SIGSEGV)
ka->sa.sa_handler = SIG_DFL;
force_sig(SIGSEGV, current);
signal_fault(regs, frame, "32bit rt signal setup");
}
arch/x86_64/ia32/ia32entry.S
View file @ e4b9e2aa
......@@ -24,9 +24,6 @@
/*
* 32bit SYSCALL instruction entry.
* It'll probably kill you because it destroys your segments.
* Should coredump here, but the next instruction will likely do
* that anyways.
*/
ENTRY(ia32_cstar_target)
movq $-ENOSYS,%rax
......@@ -117,6 +114,7 @@ ENTRY(ia32_ptregs_common)
.data
.align 8
.globl ia32_sys_call_table
ia32_sys_call_table:
.quad ni_syscall /* 0 - old "setup" system call*/
.quad sys_exit
......@@ -143,7 +141,7 @@ ia32_sys_call_table:
.quad sys_oldumount /* old_umount */
.quad sys_setuid16
.quad sys_getuid16
.quad ni_syscall /* stime */ /* 25 */
.quad sys_stime /* stime */ /* 25 */
.quad sys32_ptrace /* ptrace */
.quad sys_alarm /* XXX sign extension??? */
.quad ni_syscall /* (old)fstat */
......@@ -240,7 +238,7 @@ ia32_sys_call_table:
.quad stub32_sigreturn
.quad stub32_clone /* 120 */
.quad sys_setdomainname
.quad sys_newuname
.quad sys32_newuname
.quad sys_modify_ldt
.quad sys32_adjtimex
.quad sys32_mprotect /* 125 */
......
arch/x86_64/ia32/sys_ia32.c
View file @ e4b9e2aa
......@@ -2579,6 +2579,8 @@ int sys32_uname(struct old_utsname * name)
down_read(&uts_sem);
err=copy_to_user(name, &system_utsname, sizeof (*name));
up_read(&uts_sem);
if (current->personality == PER_LINUX32)
err |= copy_to_user(&name->machine, "i386", 5);
return err?-EFAULT:0;
}
......@@ -3125,3 +3127,6 @@ static int __init ia32_init (void)
}
__initcall(ia32_init);
extern unsigned long ia32_sys_call_table[];
EXPORT_SYMBOL(ia32_sys_call_table);
arch/x86_64/kernel/Makefile
View file @ e4b9e2aa
......@@ -12,11 +12,6 @@ obj-y := process.o semaphore.o signal.o entry.o traps.o irq.o \
pci-dma.o x8664_ksyms.o i387.o syscall.o vsyscall.o \
setup64.o bluesmoke.o bootflag.o
ifdef CONFIG_PCI
obj-y += pci-x86_64.o
obj-y += pci-pc.o pci-irq.o
endif
obj-$(CONFIG_MTRR) += mtrr.o
obj-$(CONFIG_X86_MSR) += msr.o
obj-$(CONFIG_X86_CPUID) += cpuid.o
......
arch/x86_64/kernel/bootflag.c
View file @ e4b9e2aa
......@@ -48,7 +48,7 @@ static int __init sbf_struct_valid(unsigned long tptr)
unsigned int i;
struct sbf_boot sb;
memcpy_fromio(&sb, tptr, sizeof(sb));
memcpy_fromio(&sb, (void *)tptr, sizeof(sb));
if(sb.sbf_len != 40 && sb.sbf_len != 39)
// 39 on IBM ThinkPad A21m, BIOS version 1.02b (KXET24WW; 2000-12-19).
......@@ -238,6 +238,7 @@ static int __init sbf_init(void)
rp = (unsigned long)ioremap(rp, 4096);
if(rp == 0)
continue;
if(sbf_struct_valid(rp))
{
/* Found the BOOT table and processed it */
......
arch/x86_64/kernel/early_printk.c
View file @ e4b9e2aa
......@@ -94,15 +94,20 @@ static void early_serial_write(struct console *con, const char *s, unsigned n)
static __init void early_serial_init(char *opt)
{
static int bases[] = { 0x3f8, 0x2f8 };
unsigned char c;
unsigned divisor, baud = 38400;
char *s, *e;
if (*opt == ',')
++opt;
s = strsep(&opt, ",");
if (s != NULL) {
unsigned port;
++s;
if (!strncmp(s,"0x",2))
early_serial_base = simple_strtoul(s, &e, 16);
else {
static int bases[] = { 0x3f8, 0x2f8 };
if (!strncmp(s,"ttyS",4))
s+=4;
port = simple_strtoul(s, &e, 10);
......@@ -110,12 +115,11 @@ static __init void early_serial_init(char *opt)
port = 0;
early_serial_base = bases[port];
}
}
c = inb(early_serial_base + LCR);
outb(c & ~DLAB, early_serial_base + LCR);
outb(0x3, early_serial_base + LCR); /* 8n1 */
outb(0, early_serial_base + IER); /* no interrupt */
outb(0, early_serial_base + FCR); /* no fifo */
outb(0x3, early_serial_base + LCR); /* 8n1 */
outb(0x3, early_serial_base + MCR); /* DTR + RTS */
s = strsep(&opt, ",");
......@@ -155,33 +159,55 @@ void early_printk(const char *fmt, ...)
va_end(ap);
}
static int keep_early;
int __init setup_early_printk(char *opt)
{
char *space;
char buf[256];
if (early_console_initialized)
return;
early_console_initialized = 1;
return -1;
strncpy(buf,opt,256);
buf[255] = 0;
space = strchr(buf, ' ');
if (space)
*space = 0;
if (strstr(buf,"keep"))
keep_early = 1;
if (!strncmp(opt, "serial", 6)) {
early_serial_init(opt+7);
if (!strncmp(buf, "serial", 6)) {
early_serial_init(buf + 6);
early_console = &early_serial_console;
} else if (!strncmp(opt, "vga", 3))
} else if (!strncmp(buf, "vga", 3)) {
early_console = &early_vga_console;
else
} else {
early_console = NULL;
return -1;
}
early_console_initialized = 1;
register_console(early_console);
return 0;
}
void __init disable_early_printk(void)
{
if (early_console_initialized) {
if (!early_console_initialized || !early_console)
return;
if (!keep_early) {
printk("disabling early console...\n");
unregister_console(early_console);
early_console_initialized = 0;
} else {
printk("keeping early console.\n");
}
}
/* syntax: earlyprintk=vga
earlyprintk=serial[,ttySn[,baudrate]]
Append ,keep to not disable it when the real console takes over.
Only vga or serial at a time, not both.
Currently only ttyS0 and ttyS1 are supported.
Interaction with the standard serial driver is not very good.
......
arch/x86_64/kernel/entry.S
View file @ e4b9e2aa
......@@ -40,6 +40,7 @@
#include <asm/unistd.h>
#include <asm/thread_info.h>
#include <asm/hw_irq.h>
#include <asm/errno.h>
.code64
......@@ -270,8 +271,8 @@ int_very_careful:
/* Check for syscall exit trace */
bt $TIF_SYSCALL_TRACE,%edx
jnc int_signal
movq %rsp,%rdi # &ptregs -> arg1
pushq %rdi
leaq 8(%rsp),%rdi # &ptregs -> arg1
call syscall_trace
popq %rdi
btr $TIF_SYSCALL_TRACE,%edi
......@@ -563,16 +564,17 @@ ENTRY(kernel_thread)
# rdi: flags, rsi: usp, rdx: will be &pt_regs
movq %rdx,%rdi
orq $CLONE_VM, %rdi
orq kernel_thread_flags(%rip), %rdi
movq $-1, %rsi
movq %rsp, %rdx
# clone now
call do_fork_FIXME_NOW_RETURNS_TASK_STRUCT
# save retval on the stack so it's popped before `ret`
movq %rax, RAX(%rsp)
call do_fork
xorl %edi,%edi
cmpq $-1000,%rax
cmovb %rdi,%rax
movq %rax,RAX(%rsp)
/*
* It isn't worth to check for reschedule here,
......
arch/x86_64/kernel/head.S
View file @ e4b9e2aa
......@@ -38,7 +38,11 @@ startup_32:
movl %ebx,%ebp /* Save trampoline flag */
/* First check if extended functions are implemented */
/* If the CPU doesn't support CPUID this will double fault.
* Unfortunately it is hard to check for CPUID without a stack.
*/
/* Check if extended functions are implemented */
movl $0x80000000, %eax
cpuid
cmpl $0x80000000, %eax
......@@ -157,6 +161,17 @@ reach_long64:
*/
lgdt pGDT64
/*
* Setup up a dummy PDA. this is just for some early bootup code
* that does in_interrupt()
*/
movl $MSR_GS_BASE,%ecx
movq $empty_zero_page,%rax
movq %rax,%rdx
shrq $32,%rdx
wrmsr
/* set up data segments. actually 0 would do too */
movl $__KERNEL_DS,%eax
movl %eax,%ds
movl %eax,%ss
......
arch/x86_64/kernel/head64.c
View file @ e4b9e2aa
......@@ -70,16 +70,18 @@ static void __init setup_boot_cpu_data(void)
boot_cpu_data.x86_mask = eax & 0xf;
}
extern void start_kernel(void), pda_init(int);
extern void start_kernel(void), pda_init(int), setup_early_printk(char *);
void __init x86_64_start_kernel(char * real_mode_data)
{
char *s;
clear_bss();
pda_init(0);
copy_bootdata(real_mode_data);
s = strstr(saved_command_line, "earlyprintk=");
if (s != NULL)
setup_early_printk(s+12);
setup_boot_cpu_data();
start_kernel();
}
arch/x86_64/kernel/i387.c
View file @ e4b9e2aa
......@@ -24,8 +24,6 @@
#include <asm/ptrace.h>
#include <asm/uaccess.h>
static struct i387_fxsave_struct init_fpu_env;
/*
* Called at bootup to set up the initial FPU state that is later cloned
* into all processes.
......@@ -77,6 +75,9 @@ int save_i387(struct _fpstate *buf)
bad_user_i387_struct();
}
if ((unsigned long)buf % 16)
printk("save_i387: bad fpstate %p\n",buf);
if (!tsk->used_math)
return 0;
tsk->used_math = 0; /* trigger finit */
......
arch/x86_64/kernel/i8259.c
View file @ e4b9e2aa
......@@ -12,6 +12,7 @@
#include <linux/smp_lock.h>
#include <linux/init.h>
#include <linux/kernel_stat.h>
#include <linux/device.h>
#include <asm/atomic.h>
#include <asm/system.h>
......@@ -319,6 +320,18 @@ void mask_and_ack_8259A(unsigned int irq)
}
}
static struct device device_i8259A = {
name: "i8259A",
bus_id: "0020",
};
static int __init init_8259A_devicefs(void)
{
return register_sys_device(&device_i8259A);
}
__initcall(init_8259A_devicefs);
void __init init_8259A(int auto_eoi)
{
unsigned long flags;
......
arch/x86_64/kernel/irq.c
View file @ e4b9e2aa
......@@ -163,7 +163,7 @@ int show_interrupts(struct seq_file *p, void *v)
}
seq_printf(p, "NMI: ");
for (j = 0; j < smp_num_cpus; j++)
seq_printf(p, "%10u ", nmi_count(cpu_logical_map(j)));
seq_printf(p, "%10u ", cpu_pda[cpu_logical_map(j)].__nmi_count);
seq_putc(p, '\n');
#if CONFIG_X86_LOCAL_APIC
seq_printf(p, "LOC: ");
......
arch/x86_64/kernel/ldt.c
View file @ e4b9e2aa
......@@ -8,11 +8,6 @@
* This handles calls from both 32bit and 64bit mode.
*/
/*
* FIXME:
* Need to add locking for LAR in load_gs_index.
*/
#include <linux/errno.h>
#include <linux/sched.h>
#include <linux/string.h>
......@@ -20,42 +15,164 @@
#include <linux/smp.h>
#include <linux/smp_lock.h>
#include <linux/vmalloc.h>
#include <linux/slab.h>
#include <asm/uaccess.h>
#include <asm/system.h>
#include <asm/ldt.h>
#include <asm/desc.h>
void load_gs_index(unsigned gs)
{
int access;
struct task_struct *me = current;
if (me->mm)
read_lock(&me->mm->context.ldtlock);
asm volatile("pushf\n\t"
"cli\n\t"
"swapgs\n\t"
"lar %1,%0\n\t"
"jnz 1f\n\t"
"movl %1,%%eax\n\t"
"movl %%eax,%%gs\n\t"
"jmp 2f\n\t"
"1: movl %2,%%gs\n\t"
"2: swapgs\n\t"
"popf" : "=g" (access) : "g" (gs), "r" (0) : "rax");
if (me->mm)
read_unlock(&me->mm->context.ldtlock);
}
#ifdef CONFIG_SMP /* avoids "defined but not used" warnig */
static void flush_ldt(void *mm)
{
if (current->mm)
load_LDT(&current->mm->context);
}
#endif
static int alloc_ldt(mm_context_t *pc, int mincount, int reload)
{
void *oldldt;
void *newldt;
int oldsize;
if (mincount <= pc->size)
return 0;
oldsize = pc->size;
mincount = (mincount+511)&(~511);
if (mincount*LDT_ENTRY_SIZE > PAGE_SIZE)
newldt = vmalloc(mincount*LDT_ENTRY_SIZE);
else
newldt = kmalloc(mincount*LDT_ENTRY_SIZE, GFP_KERNEL);
if (!newldt)
return -ENOMEM;
if (oldsize)
memcpy(newldt, pc->ldt, oldsize*LDT_ENTRY_SIZE);
oldldt = pc->ldt;
memset(newldt+oldsize*LDT_ENTRY_SIZE, 0, (mincount-oldsize)*LDT_ENTRY_SIZE);
wmb();
pc->ldt = newldt;
pc->size = mincount;
if (reload) {
load_LDT(pc);
#ifdef CONFIG_SMP
if (current->mm->cpu_vm_mask != (1<<smp_processor_id()))
smp_call_function(flush_ldt, 0, 1, 1);
#endif
}
wmb();
if (oldsize) {
if (oldsize*LDT_ENTRY_SIZE > PAGE_SIZE)
vfree(oldldt);
else
kfree(oldldt);
}
return 0;
}
static inline int copy_ldt(mm_context_t *new, mm_context_t *old)
{
int err = alloc_ldt(new, old->size, 0);
if (err < 0) {
printk(KERN_WARNING "ldt allocation failed\n");
new->size = 0;
return err;
}
memcpy(new->ldt, old->ldt, old->size*LDT_ENTRY_SIZE);
return 0;
}
/*
* read_ldt() is not really atomic - this is not a problem since
* synchronization of reads and writes done to the LDT has to be
* assured by user-space anyway. Writes are atomic, to protect
* the security checks done on new descriptors.
* we do not have to muck with descriptors here, that is
* done in switch_mm() as needed.
*/
int init_new_context(struct task_struct *tsk, struct mm_struct *mm)
{
struct mm_struct * old_mm;
int retval = 0;
init_MUTEX(&mm->context.sem);
mm->context.size = 0;
old_mm = current->mm;
if (old_mm && old_mm->context.size > 0) {
down(&old_mm->context.sem);
retval = copy_ldt(&mm->context, &old_mm->context);
up(&old_mm->context.sem);
}
rwlock_init(&mm->context.ldtlock);
return retval;
}
/*
* No need to lock the MM as we are the last user
*/
void release_segments(struct mm_struct *mm)
{
if (mm->context.size) {
clear_LDT();
if (mm->context.size*LDT_ENTRY_SIZE > PAGE_SIZE)
vfree(mm->context.ldt);
else
kfree(mm->context.ldt);
mm->context.size = 0;
}
}
static int read_ldt(void * ptr, unsigned long bytecount)
{
int err;
unsigned long size;
struct mm_struct * mm = current->mm;
err = 0;
if (!mm->context.segments)
goto out;
size = LDT_ENTRIES*LDT_ENTRY_SIZE;
if (!mm->context.size)
return 0;
if (bytecount > LDT_ENTRY_SIZE*LDT_ENTRIES)
bytecount = LDT_ENTRY_SIZE*LDT_ENTRIES;
down(&mm->context.sem);
size = mm->context.size*LDT_ENTRY_SIZE;
if (size > bytecount)
size = bytecount;
err = size;
if (copy_to_user(ptr, mm->context.segments, size))
err = 0;
if (copy_to_user(ptr, mm->context.ldt, size))
err = -EFAULT;
out:
up(&mm->context.sem);
if (err < 0)
return err;
if (size != bytecount) {
/* zero-fill the rest */
clear_user(ptr+size, bytecount-size);
}
return bytecount;
}
static int read_default_ldt(void * ptr, unsigned long bytecount)
{
/* Arbitary number */
/* x86-64 default LDT is all zeros */
if (bytecount > 128)
bytecount = 128;
if (clear_user(ptr, bytecount))
......@@ -94,24 +211,14 @@ static int write_ldt(void * ptr, unsigned long bytecount, int oldmode)
me->thread.gs = 0;
me->thread.fs = 0;
/*
* the GDT index of the LDT is allocated dynamically, and is
* limited by MAX_LDT_DESCRIPTORS.
*/
down_write(&mm->mmap_sem);
if (!mm->context.segments) {
void * segments = vmalloc(LDT_ENTRIES*LDT_ENTRY_SIZE);
error = -ENOMEM;
if (!segments)
down(&mm->context.sem);
if (ldt_info.entry_number >= mm->context.size) {
error = alloc_ldt(&current->mm->context, ldt_info.entry_number+1, 1);
if (error < 0)
goto out_unlock;
memset(segments, 0, LDT_ENTRIES*LDT_ENTRY_SIZE);
wmb();
mm->context.segments = segments;
mm->context.cpuvalid = 1UL << smp_processor_id();
load_LDT(mm);
}
lp = (__u32 *) ((ldt_info.entry_number << 3) + (char *) mm->context.segments);
lp = (__u32 *) ((ldt_info.entry_number << 3) + (char *) mm->context.ldt);
/* Allow LDTs to be cleared by the user. */
if (ldt_info.base_addr == 0 && ldt_info.limit == 0) {
......@@ -146,12 +253,14 @@ static int write_ldt(void * ptr, unsigned long bytecount, int oldmode)
/* Install the new entry ... */
install:
write_lock(&mm->context.ldtlock);
*lp = entry_1;
*(lp+1) = entry_2;
write_unlock(&mm->context.ldtlock);
error = 0;
out_unlock:
up_write(&mm->mmap_sem);
up(&mm->context.sem);
out:
return error;
}
......
arch/x86_64/kernel/mpparse.c
View file @ e4b9e2aa
......@@ -95,42 +95,6 @@ static int __init mpf_checksum(unsigned char *mp, int len)
return sum & 0xFF;
}
/*
* Processor encoding in an MP configuration block
*/
static char __init *mpc_family(int family,int model)
{
static char n[32];
static char *model_defs[]=
{
"80486DX","80486DX",
"80486SX","80486DX/2 or 80487",
"80486SL","80486SX/2",
"Unknown","80486DX/2-WB",
"80486DX/4","80486DX/4-WB"
};
switch (family) {
case 0x04:
if (model < 10)
return model_defs[model];
break;
case 0x05:
return("Pentium(tm)");
case 0x06:
return("Pentium(tm) Pro");
case 0x0F:
if (model == 0x0F)
return("Special controller");
}
sprintf(n,"Unknown CPU [%d:%d]",family, model);
return n;
}
static void __init MP_processor_info (struct mpc_config_processor *m)
{
int ver;
......@@ -138,10 +102,10 @@ static void __init MP_processor_info (struct mpc_config_processor *m)
if (!(m->mpc_cpuflag & CPU_ENABLED))
return;
printk("Processor #%d %s APIC version %d\n",
printk("Processor #%d %d:%d APIC version %d\n",
m->mpc_apicid,
mpc_family( (m->mpc_cpufeature & CPU_FAMILY_MASK)>>8 ,
(m->mpc_cpufeature & CPU_MODEL_MASK)>>4),
(m->mpc_cpufeature & CPU_FAMILY_MASK)>>8,
(m->mpc_cpufeature & CPU_MODEL_MASK)>>4,
m->mpc_apicver);
if (m->mpc_featureflag&(1<<0))
......
arch/x86_64/kernel/msr.c
View file @ e4b9e2aa
......@@ -41,6 +41,7 @@
#include <asm/msr.h>
#include <asm/uaccess.h>
#include <asm/system.h>
#include <asm/cpufeature.h>
/* Note: "err" is handled in a funny way below. Otherwise one version
of gcc or another breaks. */
......@@ -57,7 +58,7 @@ static inline int wrmsr_eio(u32 reg, u32 eax, u32 edx)
" jmp 2b\n"
".previous\n"
".section __ex_table,\"a\"\n"
" .align 4\n"
" .align 8\n"
" .quad 1b,3b\n"
".previous"
: "=&bDS" (err)
......@@ -236,7 +237,7 @@ static int msr_open(struct inode *inode, struct file *file)
if ( !(cpu_online_map & (1UL << cpu)) )
return -ENXIO; /* No such CPU */
if ( !test_bit(X86_FEATURE_MSR, &c->x86_capability) )
if ( !cpu_has(c, X86_FEATURE_MSR) )
return -EIO; /* MSR not supported */
return 0;
......
arch/x86_64/kernel/mtrr.c
View file @ e4b9e2aa
......@@ -139,7 +139,7 @@ static void set_mtrr_prepare (struct set_mtrr_context *ctxt)
__cli();
/* Save value of CR4 and clear Page Global Enable (bit 7) */
if (test_bit(X86_FEATURE_PGE, &boot_cpu_data.x86_capability)) {
if (cpu_has_ge) {
ctxt->cr4val = read_cr4();
write_cr4(ctxt->cr4val & ~(1UL << 7));
}
......@@ -170,7 +170,7 @@ static void set_mtrr_done (struct set_mtrr_context *ctxt)
write_cr0(read_cr0() & 0xbfffffff);
/* Restore value of CR4 */
if (test_bit(X86_FEATURE_PGE, &boot_cpu_data.x86_capability))
if (cpu_has_pge)
write_cr4 (ctxt->cr4val);
/* Re-enable interrupts locally (if enabled previously) */
......@@ -983,7 +983,7 @@ static ssize_t mtrr_write (struct file *file, const char *buf,
char *ptr;
char line[LINE_SIZE];
if (!capable (CAP_SYS_ADMIN))
if (!capable(CAP_SYS_ADMIN))
return -EPERM;
/* Can't seek (pwrite) on this device */
......@@ -1071,7 +1071,7 @@ static int mtrr_ioctl (struct inode *inode, struct file *file,
return -ENOIOCTLCMD;
case MTRRIOC_ADD_ENTRY:
if (!capable (CAP_SYS_ADMIN))
if (!capable(CAP_SYS_ADMIN))
return -EPERM;
if (copy_from_user (&sentry, (void *) arg, sizeof sentry))
return -EFAULT;
......@@ -1083,7 +1083,7 @@ static int mtrr_ioctl (struct inode *inode, struct file *file,
break;
case MTRRIOC_SET_ENTRY:
if (!capable (CAP_SYS_ADMIN))
if (!capable(CAP_SYS_ADMIN))
return -EPERM;
if (copy_from_user (&sentry, (void *) arg, sizeof sentry))
return -EFAULT;
......@@ -1093,7 +1093,7 @@ static int mtrr_ioctl (struct inode *inode, struct file *file,
break;
case MTRRIOC_DEL_ENTRY:
if (!capable (CAP_SYS_ADMIN))
if (!capable(CAP_SYS_ADMIN))
return -EPERM;
if (copy_from_user (&sentry, (void *) arg, sizeof sentry))
return -EFAULT;
......@@ -1103,7 +1103,7 @@ static int mtrr_ioctl (struct inode *inode, struct file *file,
break;
case MTRRIOC_KILL_ENTRY:
if (!capable (CAP_SYS_ADMIN))
if (!capable(CAP_SYS_ADMIN))
return -EPERM;
if (copy_from_user (&sentry, (void *) arg, sizeof sentry))
return -EFAULT;
......@@ -1134,7 +1134,7 @@ static int mtrr_ioctl (struct inode *inode, struct file *file,
break;
case MTRRIOC_ADD_PAGE_ENTRY:
if (!capable (CAP_SYS_ADMIN))
if (!capable(CAP_SYS_ADMIN))
return -EPERM;
if (copy_from_user (&sentry, (void *) arg, sizeof sentry))
return -EFAULT;
......@@ -1146,7 +1146,7 @@ static int mtrr_ioctl (struct inode *inode, struct file *file,
break;
case MTRRIOC_SET_PAGE_ENTRY:
if (!capable (CAP_SYS_ADMIN))
if (!capable(CAP_SYS_ADMIN))
return -EPERM;
if (copy_from_user (&sentry, (void *) arg, sizeof sentry))
return -EFAULT;
......@@ -1156,7 +1156,7 @@ static int mtrr_ioctl (struct inode *inode, struct file *file,
break;
case MTRRIOC_DEL_PAGE_ENTRY:
if (!capable (CAP_SYS_ADMIN))
if (!capable(CAP_SYS_ADMIN))
return -EPERM;
if (copy_from_user (&sentry, (void *) arg, sizeof sentry))
return -EFAULT;
......@@ -1166,7 +1166,7 @@ static int mtrr_ioctl (struct inode *inode, struct file *file,
break;
case MTRRIOC_KILL_PAGE_ENTRY:
if (!capable (CAP_SYS_ADMIN))
if (!capable(CAP_SYS_ADMIN))
return -EPERM;
if (copy_from_user (&sentry, (void *) arg, sizeof sentry))
return -EFAULT;
......@@ -1277,7 +1277,7 @@ static void __init mtrr_setup (void)
{
printk ("mtrr: v%s)\n", MTRR_VERSION);
if (test_bit (X86_FEATURE_MTRR, &boot_cpu_data.x86_capability)) {
if (cpu_has_mtrr) {
/* Query the width (in bits) of the physical
addressable memory on the Hammer family. */
if ((cpuid_eax (0x80000000) >= 0x80000008)) {
......
arch/x86_64/kernel/nmi.c
View file @ e4b9e2aa
......@@ -50,15 +50,19 @@ int __init check_nmi_watchdog (void)
printk(KERN_INFO "testing NMI watchdog ... ");
for (j = 0; j < NR_CPUS; ++j)
counts[j] = cpu_pda[cpu_logical_map(j)].__nmi_count;
for (j = 0; j < NR_CPUS; ++j) {
cpu = cpu_logical_map(j);
counts[cpu] = cpu_pda[cpu].__nmi_count;
}
sti();
mdelay((10*1000)/nmi_hz); // wait 10 ticks
for (j = 0; j < smp_num_cpus; j++) {
cpu = cpu_logical_map(j);
if (nmi_count(cpu) - counts[j] <= 5) {
printk("CPU#%d: NMI appears to be stuck!\n", cpu);
if (cpu_pda[cpu].__nmi_count - counts[cpu] <= 5) {
printk("CPU#%d: NMI appears to be stuck (%d)!\n",
cpu,
cpu_pda[cpu].__nmi_count);
return -1;
}
}
......
arch/x86_64/kernel/process.c
View file @ e4b9e2aa
......@@ -57,6 +57,8 @@
asmlinkage extern void ret_from_fork(void);
unsigned long kernel_thread_flags = CLONE_VM;
int hlt_counter;
/*
......@@ -318,44 +320,11 @@ void show_regs(struct pt_regs * regs)
printk("CR2: %016lx CR3: %016lx CR4: %016lx\n", cr2, cr3, cr4);
}
/*
* No need to lock the MM as we are the last user
*/
void release_segments(struct mm_struct *mm)
{
void * ldt = mm->context.segments;
/*
* free the LDT
*/
if (ldt) {
mm->context.segments = NULL;
clear_LDT();
vfree(ldt);
}
}
void load_gs_index(unsigned gs)
{
int access;
/* should load gs in syscall exit after swapgs instead */
/* XXX need to add LDT locking for SMP to protect against parallel changes */
asm volatile("pushf\n\t"
"cli\n\t"
"swapgs\n\t"
"lar %1,%0\n\t"
"jnz 1f\n\t"
"movl %1,%%eax\n\t"
"movl %%eax,%%gs\n\t"
"jmp 2f\n\t"
"1: movl %2,%%gs\n\t"
"2: swapgs\n\t"
"popf" : "=g" (access) : "g" (gs), "r" (0) : "rax");
}
#define __STR(x) #x
#define __STR2(x) __STR(x)
extern void load_gs_index(unsigned);
/*
* Free current thread data structures etc..
*/
......@@ -379,43 +348,16 @@ void flush_thread(void)
void release_thread(struct task_struct *dead_task)
{
if (dead_task->mm) {
void * ldt = dead_task->mm->context.segments;
// temporary debugging check
if (ldt) {
printk("WARNING: dead process %8s still has LDT? <%p>\n",
dead_task->comm, ldt);
if (dead_task->mm->context.size) {
printk("WARNING: dead process %8s still has LDT? <%p/%d>\n",
dead_task->comm,
dead_task->mm->context.ldt,
dead_task->mm->context.size);
BUG();
}
}
}
/*
* we do not have to muck with descriptors here, that is
* done in switch_mm() as needed.
*/
void copy_segments(struct task_struct *p, struct mm_struct *new_mm)
{
struct mm_struct * old_mm;
void *old_ldt, *ldt;
ldt = NULL;
old_mm = current->mm;
if (old_mm && (old_ldt = old_mm->context.segments) != NULL) {
/*
* Completely new LDT, we initialize it from the parent:
*/
ldt = vmalloc(LDT_ENTRIES*LDT_ENTRY_SIZE);
if (!ldt)
printk(KERN_WARNING "ldt allocation failed\n");
else
memcpy(ldt, old_ldt, LDT_ENTRIES*LDT_ENTRY_SIZE);
}
new_mm->context.segments = ldt;
new_mm->context.cpuvalid = 0UL;
return;
}
int copy_thread(int nr, unsigned long clone_flags, unsigned long rsp,
unsigned long unused,
struct task_struct * p, struct pt_regs * regs)
......
arch/x86_64/kernel/setup.c
View file @ e4b9e2aa
......@@ -28,7 +28,6 @@
#include <linux/delay.h>
#include <linux/config.h>
#include <linux/init.h>
#include <linux/apm_bios.h>
#ifdef CONFIG_BLK_DEV_RAM
#include <linux/blk.h>
#endif
......@@ -66,6 +65,8 @@ unsigned long mmu_cr4_features;
/* For PCI or other memory-mapped resources */
unsigned long pci_mem_start = 0x10000000;
unsigned long saved_video_mode;
/*
* Setup options
*/
......@@ -546,6 +547,8 @@ static inline void parse_mem_cmdline (char ** cmdline_p)
unsigned long start_pfn, end_pfn;
extern void exception_table_check(void);
void __init setup_arch(char **cmdline_p)
{
unsigned long bootmap_size, low_mem_size;
......@@ -555,6 +558,7 @@ void __init setup_arch(char **cmdline_p)
drive_info = DRIVE_INFO;
screen_info = SCREEN_INFO;
aux_device_present = AUX_DEVICE_INFO;
saved_video_mode = SAVED_VIDEO_MODE;
#ifdef CONFIG_BLK_DEV_RAM
rd_image_start = RAMDISK_FLAGS & RAMDISK_IMAGE_START_MASK;
......@@ -783,6 +787,8 @@ void __init setup_arch(char **cmdline_p)
conswitchp = &dummy_con;
#endif
#endif
exception_table_check();
}
#ifndef CONFIG_X86_TSC
......@@ -1093,7 +1099,7 @@ static int show_cpuinfo(struct seq_file *m, void *v)
else
seq_printf(m, "stepping\t: unknown\n");
if ( test_bit(X86_FEATURE_TSC, &c->x86_capability) ) {
if (cpu_has(c,X86_FEATURE_TSC)) {
seq_printf(m, "cpu MHz\t\t: %u.%03u\n",
cpu_khz / 1000, (cpu_khz % 1000));
}
......
arch/x86_64/kernel/setup64.c
View file @ e4b9e2aa
......@@ -10,6 +10,7 @@
#include <linux/kernel.h>
#include <linux/sched.h>
#include <linux/string.h>
#include <linux/bootmem.h>
#include <asm/pda.h>
#include <asm/pgtable.h>
#include <asm/processor.h>
......@@ -31,42 +32,64 @@ extern void ia32_cstar_target(void);
extern struct task_struct init_task;
extern unsigned char __per_cpu_start[], __per_cpu_end[];
struct desc_ptr gdt_descr = { 0 /* filled in */, (unsigned long) gdt_table };
struct desc_ptr idt_descr = { 256 * 16, (unsigned long) idt_table };
char boot_cpu_stack[IRQSTACKSIZE] __cacheline_aligned;
void __init setup_per_cpu_areas(void)
{
unsigned long size, i;
unsigned char *ptr;
/* Copy section for each CPU (we discard the original) */
size = ALIGN(__per_cpu_end - __per_cpu_start, SMP_CACHE_BYTES);
if (!size)
return;
ptr = alloc_bootmem(size * NR_CPUS);
for (i = 0; i < NR_CPUS; i++, ptr += size) {
cpu_pda[cpu_logical_map(i)].cpudata_offset = ptr - __per_cpu_start;
memcpy(ptr, __per_cpu_start, size);
}
}
void pda_init(int cpu)
{
pml4_t *level4;
struct x8664_pda *pda = &cpu_pda[cpu];
if (cpu == 0) {
/* others are initialized in smpboot.c */
cpu_pda[cpu].pcurrent = &init_task;
cpu_pda[cpu].irqstackptr = boot_cpu_stack;
pda->pcurrent = &init_task;
pda->irqstackptr = boot_cpu_stack;
level4 = init_level4_pgt;
} else {
cpu_pda[cpu].irqstackptr = (char *)
pda->irqstackptr = (char *)
__get_free_pages(GFP_ATOMIC, IRQSTACK_ORDER);
if (!cpu_pda[cpu].irqstackptr)
if (!pda->irqstackptr)
panic("cannot allocate irqstack for cpu %d\n", cpu);
level4 = (pml4_t *)__get_free_pages(GFP_ATOMIC, 0);
}
if (!level4)
panic("Cannot allocate top level page for cpu %d", cpu);
cpu_pda[cpu].level4_pgt = (unsigned long *)level4;
pda->level4_pgt = (unsigned long *)level4;
if (level4 != init_level4_pgt)
memcpy(level4, &init_level4_pgt, PAGE_SIZE);
set_pml4(level4 + 510, mk_kernel_pml4(__pa_symbol(boot_vmalloc_pgt)));
asm volatile("movq %0,%%cr3" :: "r" (__pa(level4)));
cpu_pda[cpu].irqstackptr += IRQSTACKSIZE-64;
cpu_pda[cpu].cpunumber = cpu;
cpu_pda[cpu].irqcount = -1;
cpu_pda[cpu].kernelstack =
pda->irqstackptr += IRQSTACKSIZE-64;
pda->cpunumber = cpu;
pda->irqcount = -1;
pda->kernelstack =
(unsigned long)stack_thread_info() - PDA_STACKOFFSET + THREAD_SIZE;
cpu_pda[cpu].me = &cpu_pda[cpu];
pda->me = pda;
pda->cpudata_offset = 0;
asm volatile("movl %0,%%fs ; movl %0,%%gs" :: "r" (0));
wrmsrl(MSR_GS_BASE, cpu_pda + cpu);
......@@ -75,7 +98,6 @@ void pda_init(int cpu)
#define EXCEPTION_STK_ORDER 0 /* >= N_EXCEPTION_STACKS*EXCEPTION_STKSZ */
char boot_exception_stacks[N_EXCEPTION_STACKS*EXCEPTION_STKSZ];
/*
* cpu_init() initializes state that is per-CPU. Some data is already
* initialized (naturally) in the bootstrap process, such as the GDT
......@@ -93,6 +115,7 @@ void __init cpu_init (void)
struct tss_struct * t = &init_tss[nr];
unsigned long v;
char *estacks;
struct task_struct *me;
/* CPU 0 is initialised in head64.c */
if (nr != 0) {
......@@ -103,6 +126,8 @@ void __init cpu_init (void)
} else
estacks = boot_exception_stacks;
me = current;
if (test_and_set_bit(nr, &cpu_initialized))
panic("CPU#%d already initialized!\n", nr);
......@@ -150,14 +175,14 @@ void __init cpu_init (void)
}
atomic_inc(&init_mm.mm_count);
current->active_mm = &init_mm;
if(current->mm)
me->active_mm = &init_mm;
if (me->mm)
BUG();
enter_lazy_tlb(&init_mm, current, nr);
enter_lazy_tlb(&init_mm, me, nr);
set_tss_desc(nr, t);
load_TR(nr);
load_LDT(&init_mm);
load_LDT(&init_mm.context);
/*
* Clear all 6 debug registers:
......
arch/x86_64/kernel/signal.c
View file @ e4b9e2aa
......@@ -22,10 +22,8 @@
#include <linux/ptrace.h>
#include <linux/unistd.h>
#include <linux/stddef.h>
#include <linux/tty.h>
#include <linux/personality.h>
#include <linux/compiler.h>
#include <linux/binfmts.h>
#include <asm/ucontext.h>
#include <asm/uaccess.h>
#include <asm/i387.h>
......@@ -86,7 +84,7 @@ struct rt_sigframe
char *pretcode;
struct ucontext uc;
struct siginfo info;
struct _fpstate fpstate __attribute__((aligned(8)));
struct _fpstate fpstate;
};
static int
......@@ -126,6 +124,7 @@ restore_sigcontext(struct pt_regs *regs, struct sigcontext *sc, unsigned long *p
{
struct _fpstate * buf;
err |= __get_user(buf, &sc->fpstate);
if (buf) {
if (verify_area(VERIFY_READ, buf, sizeof(*buf)))
goto badframe;
......@@ -147,10 +146,12 @@ asmlinkage long sys_rt_sigreturn(struct pt_regs regs)
stack_t st;
long eax;
if (verify_area(VERIFY_READ, frame, sizeof(*frame)))
if (verify_area(VERIFY_READ, frame, sizeof(*frame))) {
goto badframe;
if (__copy_from_user(&set, &frame->uc.uc_sigmask, sizeof(set)))
}
if (__copy_from_user(&set, &frame->uc.uc_sigmask, sizeof(set))) {
goto badframe;
}
sigdelsetmask(&set, ~_BLOCKABLE);
spin_lock_irq(&current->sigmask_lock);
......@@ -158,15 +159,17 @@ asmlinkage long sys_rt_sigreturn(struct pt_regs regs)
recalc_sigpending();
spin_unlock_irq(&current->sigmask_lock);
if (restore_sigcontext(&regs, &frame->uc.uc_mcontext, &eax))
if (restore_sigcontext(&regs, &frame->uc.uc_mcontext, &eax)) {
goto badframe;
}
#if DEBUG_SIG
printk("%d sigreturn rip:%lx rsp:%lx frame:%p rax:%lx\n",current->pid,regs.rip,regs.rsp,frame,eax);
#endif
if (__copy_from_user(&st, &frame->uc.uc_stack, sizeof(st)))
if (__copy_from_user(&st, &frame->uc.uc_stack, sizeof(st))) {
goto badframe;
}
/* It is more difficult to avoid calling this function than to
call it and ignore errors. */
do_sigaltstack(&st, NULL, regs.rsp);
......@@ -174,10 +177,7 @@ asmlinkage long sys_rt_sigreturn(struct pt_regs regs)
return eax;
badframe:
#if DEBUG_SIG
printk("%d bad frame %p\n",current->pid,frame);
#endif
force_sig(SIGSEGV, current);
signal_fault(&regs,frame,"sigreturn");
return 0;
}
......@@ -233,8 +233,8 @@ setup_sigcontext(struct sigcontext *sc, struct _fpstate *fpstate,
/*
* Determine which stack to use..
*/
static inline void *
get_sigframe(struct k_sigaction *ka, struct pt_regs * regs, size_t frame_size)
static inline struct rt_sigframe *
get_sigframe(struct k_sigaction *ka, struct pt_regs * regs)
{
unsigned long rsp;
......@@ -247,15 +247,10 @@ get_sigframe(struct k_sigaction *ka, struct pt_regs * regs, size_t frame_size)
rsp = current->sas_ss_sp + current->sas_ss_size;
}
{
extern void bad_sigframe(void);
/* beginning of sigframe is 8 bytes misaligned, but fpstate
must end up on a 16byte boundary */
if ((offsetof(struct rt_sigframe, fpstate) & 16) != 0)
bad_sigframe();
}
rsp = (rsp - sizeof(struct _fpstate)) & ~(15UL);
rsp -= offsetof(struct rt_sigframe, fpstate);
return (void *)((rsp - frame_size) & ~(15UL)) - 8;
return (struct rt_sigframe *) rsp;
}
static void setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
......@@ -264,16 +259,17 @@ static void setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
struct rt_sigframe *frame;
int err = 0;
frame = get_sigframe(ka, regs, sizeof(*frame));
frame = get_sigframe(ka, regs);
if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame)))
goto give_sigsegv;
if (ka->sa.sa_flags & SA_SIGINFO) {
err |= copy_siginfo_to_user(&frame->info, info);
if (err)
if (err) {
goto give_sigsegv;
}
}
/* Create the ucontext. */
err |= __put_user(0, &frame->uc.uc_flags);
......@@ -285,9 +281,10 @@ static void setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
err |= setup_sigcontext(&frame->uc.uc_mcontext, &frame->fpstate,
regs, set->sig[0]);
err |= __copy_to_user(&frame->uc.uc_sigmask, set, sizeof(*set));
if (err)
goto give_sigsegv;
if (err) {
goto give_sigsegv;
}
/* Set up to return from userspace. If provided, use a stub
already in userspace. */
......@@ -299,8 +296,10 @@ static void setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
goto give_sigsegv;
}
if (err)
if (err) {
printk("fault 3\n");
goto give_sigsegv;
}
#if DEBUG_SIG
printk("%d old rip %lx old rsp %lx old rax %lx\n", current->pid,regs->rip,regs->rsp,regs->rax);
......@@ -337,7 +336,7 @@ static void setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
give_sigsegv:
if (sig == SIGSEGV)
ka->sa.sa_handler = SIG_DFL;
force_sig(SIGSEGV, current);
signal_fault(regs,frame,"signal setup");
}
/*
......@@ -459,3 +458,15 @@ void do_notify_resume(struct pt_regs *regs, sigset_t *oldset, __u32 thread_info_
if (thread_info_flags & _TIF_SIGPENDING)
do_signal(regs,oldset);
}
extern int exception_trace;
void signal_fault(struct pt_regs *regs, void *frame, char *where)
{
struct task_struct *me = current;
if (exception_trace)
printk("%s[%d] bad frame in %s frame:%p rip:%lx rsp:%lx orax:%lx\n",
me->comm,me->pid,where,frame,regs->rip,regs->rsp,regs->orig_rax);
force_sig(SIGSEGV, me);
}
arch/x86_64/kernel/smp.c
View file @ e4b9e2aa
......@@ -150,6 +150,7 @@ static void inline leave_mm (unsigned long cpu)
if (cpu_tlbstate[cpu].state == TLBSTATE_OK)
BUG();
clear_bit(cpu, &cpu_tlbstate[cpu].active_mm->cpu_vm_mask);
__flush_tlb();
}
/*
......@@ -200,10 +201,12 @@ static void inline leave_mm (unsigned long cpu)
asmlinkage void smp_invalidate_interrupt (void)
{
unsigned long cpu = smp_processor_id();
unsigned long cpu;
cpu = get_cpu();
if (!test_bit(cpu, &flush_cpumask))
return;
goto out;
/*
* This was a BUG() but until someone can quote me the
* line from the intel manual that guarantees an IPI to
......@@ -224,6 +227,9 @@ asmlinkage void smp_invalidate_interrupt (void)
}
ack_APIC_irq();
clear_bit(cpu, &flush_cpumask);
out:
put_cpu();
}
static void flush_tlb_others (unsigned long cpumask, struct mm_struct *mm,
......@@ -273,16 +279,23 @@ static void flush_tlb_others (unsigned long cpumask, struct mm_struct *mm,
void flush_tlb_current_task(void)
{
struct mm_struct *mm = current->mm;
unsigned long cpu_mask = mm->cpu_vm_mask & ~(1 << smp_processor_id());
unsigned long cpu_mask;
preempt_disable();
cpu_mask = mm->cpu_vm_mask & ~(1UL << smp_processor_id());
local_flush_tlb();
if (cpu_mask)
flush_tlb_others(cpu_mask, mm, FLUSH_ALL);
preempt_enable();
}
void flush_tlb_mm (struct mm_struct * mm)
{
unsigned long cpu_mask = mm->cpu_vm_mask & ~(1 << smp_processor_id());
unsigned long cpu_mask;
preempt_disable();
cpu_mask = mm->cpu_vm_mask & ~(1UL << smp_processor_id());
if (current->active_mm == mm) {
if (current->mm)
......@@ -292,12 +305,17 @@ void flush_tlb_mm (struct mm_struct * mm)
}
if (cpu_mask)
flush_tlb_others(cpu_mask, mm, FLUSH_ALL);
preempt_enable();
}
void flush_tlb_page(struct vm_area_struct * vma, unsigned long va)
{
struct mm_struct *mm = vma->vm_mm;
unsigned long cpu_mask = mm->cpu_vm_mask & ~(1 << smp_processor_id());
unsigned long cpu_mask;
preempt_disable();
cpu_mask = mm->cpu_vm_mask & ~(1UL << smp_processor_id());
if (current->active_mm == mm) {
if(current->mm)
......@@ -308,6 +326,8 @@ void flush_tlb_page(struct vm_area_struct * vma, unsigned long va)
if (cpu_mask)
flush_tlb_others(cpu_mask, mm, va);
preempt_enable();
}
static inline void do_flush_tlb_all_local(void)
......
arch/x86_64/kernel/smpboot.c
View file @ e4b9e2aa
......@@ -50,9 +50,6 @@
#include <asm/kdebug.h>
#include <asm/tlbflush.h>
/* Set if we find a B stepping CPU */
static int smp_b_stepping;
/* Setup configured maximum number of CPUs to activate */
static int max_cpus = -1;
......@@ -151,17 +148,6 @@ void __init smp_store_cpu_info(int id)
*c = boot_cpu_data;
identify_cpu(c);
/*
* Mask B, Pentium, but not Pentium MMX
*/
if (c->x86_vendor == X86_VENDOR_INTEL &&
c->x86 == 5 &&
c->x86_mask >= 1 && c->x86_mask <= 4 &&
c->x86_model <= 3)
/*
* Remember we have B step Pentia with bugs
*/
smp_b_stepping = 1;
}
/*
......@@ -772,7 +758,7 @@ unsigned long cache_decay_ticks;
static void smp_tune_scheduling (void)
{
unsigned long cachesize; /* kB */
unsigned long bandwidth = 350; /* MB/s */
unsigned long bandwidth = 1000; /* MB/s */
/*
* Rough estimation for SMP scheduling, this is the number of
* cycles it takes for a fully memory-limited process to flush
......@@ -883,8 +869,7 @@ void __init smp_boot_cpus(void)
/*
* If we couldn't find a local APIC, then get out of here now!
*/
if (APIC_INTEGRATED(apic_version[boot_cpu_id]) &&
!test_bit(X86_FEATURE_APIC, boot_cpu_data.x86_capability)) {
if (APIC_INTEGRATED(apic_version[boot_cpu_id]) && !cpu_has_apic) {
printk(KERN_ERR "BIOS bug, local APIC #%d not detected!...\n",
boot_cpu_id);
printk(KERN_ERR "... forcing use of dummy APIC emulation. (tell your hw vendor)\n");
......@@ -965,7 +950,7 @@ void __init smp_boot_cpus(void)
Dprintk("Before bogomips.\n");
if (!cpucount) {
printk(KERN_ERR "Error: only one processor found.\n");
printk(KERN_INFO "Only one processor found.\n");
} else {
unsigned long bogosum = 0;
for (cpu = 0; cpu < NR_CPUS; cpu++)
......@@ -979,8 +964,6 @@ void __init smp_boot_cpus(void)
}
smp_num_cpus = cpucount + 1;
if (smp_b_stepping)
printk(KERN_WARNING "WARNING: SMP operation may be unreliable with B stepping processors.\n");
Dprintk("Boot done.\n");
/*
......
arch/x86_64/kernel/sys_x86_64.c
View file @ e4b9e2aa
......@@ -14,6 +14,7 @@
#include <linux/mman.h>
#include <linux/file.h>
#include <linux/utsname.h>
#include <linux/personality.h>
#include <asm/uaccess.h>
#include <asm/ipc.h>
......@@ -22,7 +23,7 @@
* sys_pipe() is the normal C calling standard for creating
* a pipe. It's not the way Unix traditionally does this, though.
*/
asmlinkage long sys_pipe(unsigned long * fildes)
asmlinkage long sys_pipe(int *fildes)
{
int fd[2];
int error;
......@@ -93,18 +94,15 @@ unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr, unsi
}
}
/*
* Old cruft
*/
asmlinkage long sys_uname(struct old_utsname * name)
asmlinkage long sys_uname(struct new_utsname * name)
{
int err;
if (!name)
return -EFAULT;
down_read(&uts_sem);
err=copy_to_user(name, &system_utsname, sizeof (*name));
err = copy_to_user(name, &system_utsname, sizeof (*name));
up_read(&uts_sem);
return err?-EFAULT:0;
if (current->personality == PER_LINUX32)
err |= copy_to_user(&name->machine, "i386", 5);
return err ? -EFAULT : 0;
}
asmlinkage long sys_pause(void)
......
arch/x86_64/kernel/time.c
View file @ e4b9e2aa
......@@ -41,6 +41,7 @@
#include <linux/delay.h>
#include <linux/init.h>
#include <linux/smp.h>
#include <linux/device.h>
#include <asm/io.h>
#include <asm/smp.h>
......@@ -57,12 +58,8 @@
#include <asm/fixmap.h>
/*
* for x86_do_profile()
*/
#include <linux/irq.h>
unsigned int cpu_khz; /* Detected as we calibrate the TSC */
/* Number of usecs that the last interrupt was delayed */
......@@ -79,9 +76,9 @@ unsigned int __fast_gettimeoffset_quotient __section_fast_gettimeoffset_quotient
extern rwlock_t xtime_lock;
struct timeval __xtime __section_xtime;
volatile unsigned long __jiffies __section_jiffies;
unsigned long __wall_jiffies __section_wall_jiffies;
struct timezone __sys_tz __section_sys_tz;
volatile unsigned long __jiffies __section_jiffies;
spinlock_t rtc_lock = SPIN_LOCK_UNLOCKED;
......@@ -527,3 +524,15 @@ void __init time_init(void)
setup_irq(0, &irq0);
}
static struct device device_i8253 = {
name: "i8253",
bus_id: "0040",
};
static int time_init_driverfs(void)
{
return register_sys_device(&device_i8253);
}
__initcall(time_init_driverfs);
arch/x86_64/kernel/traps.c
View file @ e4b9e2aa
......@@ -77,7 +77,7 @@ asmlinkage void machine_check(void);
asmlinkage void spurious_interrupt_bug(void);
asmlinkage void call_debug(void);
extern char iret_address[];
extern int exception_trace;
struct notifier_block *die_chain;
......@@ -172,6 +172,8 @@ void show_trace(unsigned long *stack)
i = 1;
if (stack >= irqstack && stack < irqstack_end) {
unsigned long *tstack;
printk("<IRQ> ");
while (stack < irqstack_end) {
addr = *stack++;
/*
......@@ -197,13 +199,11 @@ void show_trace(unsigned long *stack)
tstack = (unsigned long *)(current_thread_info()+1);
if (stack < tstack || (char*)stack > (char*)tstack+THREAD_SIZE)
printk("\n" KERN_DEBUG
"no stack at the end of irqstack; stack:%lx, curstack %lx\n",
"no stack at the end of irqstack; stack:%p, curstack %p\n",
stack, tstack);
#endif
}
while (((long) stack & (THREAD_SIZE-1)) != 0) {
addr = *stack++;
if (kernel_text_address(addr)) {
......@@ -263,7 +263,7 @@ void show_stack(unsigned long * rsp)
void show_registers(struct pt_regs *regs)
{
int i;
int in_kernel = 1;
int in_kernel = (regs->cs & 3) == 0;
unsigned long rsp;
#ifdef CONFIG_SMP
/* For SMP should get the APIC id here, just to protect against corrupted GS */
......@@ -273,11 +273,8 @@ void show_registers(struct pt_regs *regs)
#endif
struct task_struct *cur = cpu_pda[cpu].pcurrent;
rsp = (unsigned long) (&regs->rsp);
if (regs->rsp < TASK_SIZE) {
in_kernel = 0;
rsp = regs->rsp;
}
printk("CPU %d ", cpu);
show_regs(regs);
printk("Process %s (pid: %d, stackpage=%08lx)\n",
......@@ -383,7 +380,7 @@ static void do_trap(int trapnr, int signr, char *str,
if ((regs->cs & 3) != 0) {
struct task_struct *tsk = current;
if (trapnr != 3)
if (exception_trace && trapnr != 3)
printk("%s[%d] trap %s at rip:%lx rsp:%lx err:%lx\n",
tsk->comm, tsk->pid, str, regs->rip, regs->rsp, error_code);
......@@ -456,9 +453,14 @@ extern void dump_pagetable(unsigned long);
asmlinkage void do_general_protection(struct pt_regs * regs, long error_code)
{
if ((regs->cs & 3)!=0) {
current->thread.error_code = error_code;
current->thread.trap_no = 13;
force_sig(SIGSEGV, current);
struct task_struct *tsk = current;
if (exception_trace)
printk("%s[%d] #gp at rip:%lx rsp:%lx err:%lx\n",
tsk->comm, tsk->pid, regs->rip, regs->rsp, error_code);
tsk->thread.error_code = error_code;
tsk->thread.trap_no = 13;
force_sig(SIGSEGV, tsk);
return;
}
......@@ -509,8 +511,7 @@ asmlinkage void do_nmi(struct pt_regs * regs)
{
unsigned char reason = inb(0x61);
++nmi_count(smp_processor_id());
add_pda(__nmi_count,1);
if (!(reason & 0xc0)) {
#if CONFIG_X86_LOCAL_APIC
/*
......
arch/x86_64/kernel/vsyscall.c
View file @ e4b9e2aa
......@@ -47,14 +47,14 @@
#define __vsyscall(nr) __attribute__ ((unused,__section__(".vsyscall_" #nr)))
//#define NO_VSYSCALL 1
#define NO_VSYSCALL 1
#ifdef NO_VSYSCALL
#include <asm/unistd.h>
static int errno __section_vxtime_sequence;
__syscall2(static inline int,int,gettimeofday,struct timeval *,tv,struct timezone *,tz)
static inline _syscall2(int,gettimeofday,struct timeval *,tv,struct timezone *,tz)
#else
static inline void timeval_normalize(struct timeval * tv)
......@@ -148,28 +148,11 @@ static int __vsyscall(0) vgettimeofday(struct timeval * tv, struct timezone * tz
static time_t __vsyscall(1) vtime(time_t * t)
{
#ifdef NO_VSYSCALL
struct timeval tv;
gettimeofday(&tv,NULL);
if (t) *t = tv.tv_sec;
return tv.tv_sec;
#else
long sequence;
time_t __time;
do {
sequence = __vxtime_sequence[1];
rmb();
__time = __xtime.tv_sec;
rmb();
} while (sequence != __vxtime_sequence[0]);
vgettimeofday(&tv,NULL);
if (t)
*t = __time;
return __time;
#endif
*t = tv.tv_sec;
return tv.tv_sec;
}
static long __vsyscall(2) venosys_0(void)
......
arch/x86_64/kernel/x8664_ksyms.c
View file @ e4b9e2aa
......@@ -88,8 +88,9 @@ EXPORT_SYMBOL(strncpy_from_user);
EXPORT_SYMBOL(__strncpy_from_user);
EXPORT_SYMBOL(clear_user);
EXPORT_SYMBOL(__clear_user);
EXPORT_SYMBOL(__generic_copy_from_user);
EXPORT_SYMBOL(__generic_copy_to_user);
EXPORT_SYMBOL(copy_user_generic);
EXPORT_SYMBOL(copy_from_user);
EXPORT_SYMBOL(copy_to_user);
EXPORT_SYMBOL(strnlen_user);
EXPORT_SYMBOL(pci_alloc_consistent);
......
arch/x86_64/lib/Makefile
View file @ e4b9e2aa
#
# Makefile for x86_64-specific library files..
# Makefile for x86_64-specific library files.
#
USE_STANDARD_AS_RULE := true
EXTRA_CFLAGS_csum-partial.o := -funroll-loops
L_TARGET = lib.a
obj-y = generic-checksum.o old-checksum.o delay.o \
obj-y = csum-partial.o csum-copy.o csum-wrappers.o delay.o \
usercopy.o getuser.o putuser.o \
checksum_copy.o thunk.o mmx.o
thunk.o io.o clear_page.o copy_page.o
obj-y += memcpy.o
obj-y += memmove.o
#obj-y += memset.o
obj-y += copy_user.o
export-objs := io.o csum-wrappers.o csum-partial.o
obj-$(CONFIG_IO_DEBUG) += iodebug.o
obj-$(CONFIG_HAVE_DEC_LOCK) += dec_and_lock.o
......
arch/x86_64/lib/checksum_copy.S deleted 100644 → 0
View file @ cc9af0c5
/*
* INET An implementation of the TCP/IP protocol suite for the LINUX
* operating system. INET is implemented using the BSD Socket
* interface as the means of communication with the user level.
*
* IP/TCP/UDP checksumming routines
*
* Authors: Jorge Cwik, <jorge@laser.satlink.net>
* Arnt Gulbrandsen, <agulbra@nvg.unit.no>
* Tom May, <ftom@netcom.com>
* Pentium Pro/II routines:
* Alexander Kjeldaas <astor@guardian.no>
* Finn Arne Gangstad <finnag@guardian.no>
* Lots of code moved from tcp.c and ip.c; see those files
* for more names.
*
* Changes: Ingo Molnar, converted csum_partial_copy() to 2.1 exception
* handling.
* Andi Kleen, add zeroing on error
* converted to pure assembler
* Andi Kleen initial raw port to x86-64
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version
* 2 of the License, or (at your option) any later version.
*/
#include <linux/config.h>
#include <asm/errno.h>
/* Version for PentiumII/PPro ported to x86-64. Still very raw and
does not exploit 64bit. */
#define SRC(y...) \
9999: y; \
.section __ex_table, "a"; \
.quad 9999b, 6001f ; \
.previous
#define DST(y...) \
9999: y; \
.section __ex_table, "a"; \
.quad 9999b, 6002f ; \
.previous
#define ROUND1(x) \
SRC(movl x(%rsi), %ebx ) ; \
addl %ebx, %eax ; \
DST(movl %ebx, x(%rdi) ) ;
#define ROUND(x) \
SRC(movl x(%rsi), %ebx ) ; \
adcl %ebx, %eax ; \
DST(movl %ebx, x(%rdi) ) ;
#define ARGBASE 0
/*
asmlinkage unsigned int csum_partial_copy_generic( const char *src, char *dst, int len, int sum,
int *src_err_ptr, int *dst_err_ptr);
rdi .. src
rsi .. dst (copy in r12)
rdx .. len (copy in r10)
rcx .. sum
r8 .. src_err_ptr
r9 .. dst_err_ptr
OPTIMIZEME: this routine should take advantage of checksumming 64bits at a time
*/
.globl csum_partial_copy_generic
csum_partial_copy_generic:
pushq %r10
pushq %r12
pushq %rbx
pushq %rbp
xchgq %rsi, %rdi
movq %rdx, %r10
movq %rsi, %r12
movq %rcx, %rax
movq %rdx, %rcx # And now it looks like PII case
movl %ecx, %ebx
movl %esi, %edx
shrl $6, %ecx
andl $0x3c, %ebx
negq %rbx
subq %rbx, %rsi
subq %rbx, %rdi
lea 3f(%rbx,%rbx), %rbx
testq %rsi, %rsi
jmp *%rbx
1: addq $64,%rsi
addq $64,%rdi
ROUND1(-64) ROUND(-60) ROUND(-56) ROUND(-52)
ROUND (-48) ROUND(-44) ROUND(-40) ROUND(-36)
ROUND (-32) ROUND(-28) ROUND(-24) ROUND(-20)
ROUND (-16) ROUND(-12) ROUND(-8) ROUND(-4)
3: adcl $0,%eax
addl $64,%edx
dec %ecx
jge 1b
4: movq %r10,%rdx
andl $3, %edx
jz 7f
cmpl $2, %edx
jb 5f
SRC( movw (%rsi), %dx )
leaq 2(%rsi), %rsi
DST( movw %dx, (%rdi) )
leaq 2(%rdi), %rdi
je 6f
shll $16,%edx
5:
SRC( movb (%rsi), %dl )
DST( movb %dl, (%rdi) )
6: addl %edx, %eax
adcl $0, %eax
7:
.section .fixup, "ax"
6001:
movl $-EFAULT, (%r8)
# zero the complete destination (computing the rest is too much work)
movq %r12,%rdi # dst
movq %r10,%rcx # len
xorl %eax,%eax
rep; stosb
jmp 7b
6002: movl $-EFAULT,(%r9)
jmp 7b
.previous
popq %rbp
popq %rbx
popq %r12
popq %r10
ret
#undef ROUND
#undef ROUND1
arch/x86_64/lib/clear_page.S 0 → 100644
View file @ e4b9e2aa
/*
* Copyright 2002 Andi Kleen, SuSE Labs.
*/
#include <linux/linkage.h>
/*
* Zero a page.
* rdi page
*/
ENTRY(clear_page)
xorl %eax,%eax
movl $4096/128,%ecx
movl $128,%edx
loop:
#define PUT(x) movnti %rax,x*8(%rdi)
PUT(0)
PUT(1)
PUT(2)
PUT(3)
PUT(4)
PUT(5)
PUT(6)
PUT(7)
PUT(8)
PUT(9)
PUT(10)
PUT(11)
PUT(12)
PUT(13)
PUT(14)
PUT(15)
addq %rdx,%rdi
decl %ecx
jnz loop
sfence
ret
arch/x86_64/lib/copy_page.S 0 → 100644
View file @ e4b9e2aa
/*
* Copyright 2002 Andi Kleen, SuSE Labs.
*/
#include <linux/linkage.h>
#include <linux/config.h>
#ifdef CONFIG_PREEMPT
#warning "check your fpu context saving!"
#endif
/*
* Copy a page.
*
* rdi destination page
* rsi source page
*
* src/dst must be aligned to 16 bytes.
*
* Warning: in case of super lazy FP save this needs to be preempt_stop
*/
ENTRY(copy_page)
prefetchnta (%rsi)
prefetchnta 64(%rsi)
movq %rsp,%rax
subq $16*4,%rsp
andq $~15,%rsp
movdqa %xmm0,(%rsp)
movdqa %xmm1,16(%rsp)
movdqa %xmm2,32(%rsp)
movdqa %xmm3,48(%rsp)
movl $(4096/128)-2,%ecx
movl $128,%edx
loop:
prefetchnta (%rsi)
prefetchnta 64(%rsi)
loop_no_prefetch:
movdqa (%rsi),%xmm0
movdqa 1*16(%rsi),%xmm1
movdqa 2*16(%rsi),%xmm2
movdqa 3*16(%rsi),%xmm3
movntdq %xmm0,(%rdi)
movntdq %xmm1,16(%rdi)
movntdq %xmm2,2*16(%rdi)
movntdq %xmm3,3*16(%rdi)
movdqa 4*16(%rsi),%xmm0
movdqa 5*16(%rsi),%xmm1
movdqa 6*16(%rsi),%xmm2
movdqa 7*16(%rsi),%xmm3
movntdq %xmm0,4*16(%rdi)
movntdq %xmm1,5*16(%rdi)
movntdq %xmm2,6*16(%rdi)
movntdq %xmm3,7*16(%rdi)
addq %rdx,%rdi
addq %rdx,%rsi
decl %ecx
jns loop
cmpl $-1,%ecx
je loop_no_prefetch
sfence
movdqa (%rsp),%xmm0
movdqa 16(%rsp),%xmm1
movdqa 32(%rsp),%xmm2
movdqa 48(%rsp),%xmm3
movq %rax,%rsp
ret
arch/x86_64/lib/copy_user.S 0 → 100644
View file @ e4b9e2aa
/* Copyright 2002 Andi Kleen, SuSE Labs.
* Subject to the GNU Public License v2.
*
* Functions to copy from and to user space.
*/
#define FIX_ALIGNMENT 1
#include <asm/thread_info.h>
#include <asm/offset.h>
/* Standard copy_to_user with segment limit checking */
.globl copy_to_user
.p2align
copy_to_user:
GET_THREAD_INFO(%rax)
movq %rdi,%rcx
addq %rdx,%rcx
jc bad_to_user
cmpq threadinfo_addr_limit(%rax),%rcx
jae bad_to_user
jmp copy_user_generic
/* Standard copy_from_user with segment limit checking */
.globl copy_from_user
.p2align
copy_from_user:
GET_THREAD_INFO(%rax)
movq %rsi,%rcx
addq %rdx,%rcx
jc bad_from_user
cmpq threadinfo_addr_limit(%rax),%rcx
jae bad_from_user
/* FALL THROUGH to copy_user_generic */
.section .fixup,"ax"
/* must zero dest */
bad_from_user:
movl %edx,%ecx
xorl %eax,%eax
rep
stosb
bad_to_user:
movl %edx,%eax
ret
.previous
/*
* copy_user_generic - memory copy with exception handling.
*
* Input:
* rdi destination
* rsi source
* rdx count
*
* Output:
* eax uncopied bytes or 0 if successfull.
*/
.globl copy_user_generic
copy_user_generic:
/* Put the first cacheline into cache. This should handle
the small movements in ioctls etc., but not penalize the bigger
filesystem data copies too much. */
pushq %rbx
prefetcht0 (%rsi)
xorl %eax,%eax /*zero for the exception handler */
#ifdef FIX_ALIGNMENT
/* check for bad alignment of destination */
movl %edi,%ecx
andl $7,%ecx
jnz bad_alignment
after_bad_alignment:
#endif
movq %rdx,%rcx
movl $64,%ebx
shrq $6,%rdx
decq %rdx
js handle_tail
jz loop_no_prefetch
loop:
prefetchnta 64(%rsi)
loop_no_prefetch:
s1: movq (%rsi),%r11
s2: movq 1*8(%rsi),%r8
s3: movq 2*8(%rsi),%r9
s4: movq 3*8(%rsi),%r10
d1: movnti %r11,(%rdi)
d2: movnti %r8,1*8(%rdi)
d3: movnti %r9,2*8(%rdi)
d4: movnti %r10,3*8(%rdi)
s5: movq 4*8(%rsi),%r11
s6: movq 5*8(%rsi),%r8
s7: movq 6*8(%rsi),%r9
s8: movq 7*8(%rsi),%r10
d5: movnti %r11,4*8(%rdi)
d6: movnti %r8,5*8(%rdi)
d7: movnti %r9,6*8(%rdi)
d8: movnti %r10,7*8(%rdi)
addq %rbx,%rsi
addq %rbx,%rdi
decq %rdx
jz loop_no_prefetch
jns loop
handle_tail:
movl %ecx,%edx
andl $63,%ecx
shrl $3,%ecx
jz handle_7
movl $8,%ebx
loop_8:
s9: movq (%rsi),%r8
d9: movnti %r8,(%rdi)
addq %rbx,%rdi
addq %rbx,%rsi
loop loop_8
handle_7:
movl %edx,%ecx
andl $7,%ecx
jz ende
loop_1:
s10: movb (%rsi),%bl
d10: movb %bl,(%rdi)
incq %rdi
incq %rsi
loop loop_1
ende:
sfence
popq %rbx
ret
#ifdef FIX_ALIGNMENT
/* align destination */
bad_alignment:
movl $8,%r9d
subl %ecx,%r9d
movl %r9d,%ecx
subq %r9,%rdx
jz small_align
js small_align
align_1:
s11: movb (%rsi),%bl
d11: movb %bl,(%rdi)
incq %rsi
incq %rdi
loop align_1
jmp after_bad_alignment
small_align:
addq %r9,%rdx
jmp handle_7
#endif
/* table sorted by exception address */
.section __ex_table,"a"
.align 8
.quad s1,s1e
.quad s2,s2e
.quad s3,s3e
.quad s4,s4e
.quad d1,s1e
.quad d2,s2e
.quad d3,s3e
.quad d4,s4e
.quad s5,s5e
.quad s6,s6e
.quad s7,s7e
.quad s8,s8e
.quad d5,s5e
.quad d6,s6e
.quad d7,s7e
.quad d8,s8e
.quad s9,e_quad
.quad d9,e_quad
.quad s10,e_byte
.quad d10,e_byte
#ifdef FIX_ALIGNMENT
.quad s11,e_byte
.quad d11,e_byte
#endif
.quad e5,e_zero
.previous
/* compute 64-offset for main loop. 8 bytes accuracy with error on the
pessimistic side. this is gross. it would be better to fix the
interface. */
/* eax: zero, ebx: 64 */
s1e: addl $8,%eax
s2e: addl $8,%eax
s3e: addl $8,%eax
s4e: addl $8,%eax
s5e: addl $8,%eax
s6e: addl $8,%eax
s7e: addl $8,%eax
s8e: addl $8,%eax
addq %rbx,%rdi /* +64 */
subq %rax,%rdi /* correct destination with computed offset */
shlq $6,%rdx /* loop counter * 64 (stride length) */
addq %rax,%rdx /* add offset to loopcnt */
andl $63,%ecx /* remaining bytes */
addq %rcx,%rdx /* add them */
jmp zero_rest
/* exception on quad word loop in tail handling */
/* ecx: loopcnt/8, %edx: length, rdi: correct */
e_quad:
shll $3,%ecx
andl $7,%edx
addl %ecx,%edx
/* edx: bytes to zero, rdi: dest, eax:zero */
zero_rest:
movq %rdx,%rcx
e_byte:
xorl %eax,%eax
e5: rep
stosb
/* when there is another exception while zeroing the rest just return */
e_zero:
movq %rdx,%rax
jmp ende
arch/x86_64/lib/csum-copy.S 0 → 100644
View file @ e4b9e2aa
/*
* Copyright 2002 Andi Kleen
*
* This file is subject to the terms and conditions of the GNU General Public
* License. See the file COPYING in the main directory of this archive
* for more details. No warranty for anything given at all.
*/
#include <linux/linkage.h>
#include <asm/errno.h>
// #define FIX_ALIGNMENT 1
/*
* Checksum copy with exception handling.
* On exceptions src_err_ptr or dst_err_ptr is set to -EFAULT and the
* destination is zeroed.
*
* Input
* rdi source
* rsi destination
* edx len (32bit)
* ecx sum (32bit)
* r8 src_err_ptr (int)
* r9 dst_err_ptr (int)
*
* Output
* eax 64bit sum. undefined in case of exception.
*
* Wrappers need to take care of valid exception sum and zeroing.
*/
.macro source
10:
.section __ex_table,"a"
.align 8
.quad 10b,bad_source
.previous
.endm
.macro dest
20:
.section __ex_table,"a"
.align 8
.quad 20b,bad_dest
.previous
.endm
.globl csum_partial_copy_generic
.p2align
csum_partial_copy_generic:
prefetchnta (%rdi)
pushq %rbx
pushq %r12
pushq %r14
pushq %r15
movq %r8,%r14
movq %r9,%r15
movl %ecx,%eax
movl %edx,%ecx
#ifdef FIX_ALIGNMENT
/* align source to 8 bytes */
movl %edi,%r8d
andl $7,%r8d
jnz bad_alignment
after_bad_alignment:
#endif
movl $64,%r10d
xorl %r9d,%r9d
movq %rcx,%r12
shrq $6,%r12
/* loopcounter is maintained as one less to test efficiently for the
previous to last iteration. This is needed to stop the prefetching. */
decq %r12
js handle_tail /* < 64 */
jz loop_no_prefetch /* = 64 + X */
/* main loop. clear in 64 byte blocks */
/* tries hard not to prefetch over the boundary */
/* r10: 64, r9: zero, r8: temp2, rbx: temp1, rax: sum, rcx: saved length */
/* r11: temp3, rdx: temp4, r12 loopcnt */
.p2align
loop:
/* Could prefetch more than one loop, but then it would be even
trickier to avoid prefetching over the boundary. The hardware prefetch
should take care of this anyways. The reason for this prefetch is
just the non temporal hint to avoid cache pollution. Hopefully this
will be handled properly by the hardware. */
prefetchnta 64(%rdi)
loop_no_prefetch:
source
movq (%rdi),%rbx
source
movq 8(%rdi),%r8
source
movq 16(%rdi),%r11
source
movq 24(%rdi),%rdx
dest
movnti %rbx,(%rsi)
dest
movnti %r8,8(%rsi)
dest
movnti %r11,16(%rsi)
dest
movnti %rdx,24(%rsi)
addq %rbx,%rax
adcq %r8,%rax
adcq %r11,%rax
adcq %rdx,%rax
source
movq 32(%rdi),%rbx
source
movq 40(%rdi),%r8
source
movq 48(%rdi),%r11
source
movq 56(%rdi),%rdx
dest
movnti %rbx,32(%rsi)
dest
movnti %r8,40(%rsi)
dest
movnti %r11,48(%rsi)
dest
movnti %rdx,56(%rsi)
adcq %rbx,%rax
adcq %r8,%rax
adcq %r11,%rax
adcq %rdx,%rax
adcq %r9,%rax /* add in carry */
addq %r10,%rdi
addq %r10,%rsi
decq %r12
jz loop_no_prefetch /* previous to last iteration? */
jns loop
/* do last upto 56 bytes */
handle_tail:
/* ecx: count */
movl %ecx,%r10d
andl $63,%ecx
shrl $3,%ecx
jz fold
clc
movl $8,%edx
loop_8:
source
movq (%rdi),%rbx
adcq %rbx,%rax
dest
movnti %rbx,(%rsi)
leaq (%rsi,%rdx),%rsi /* preserve carry */
leaq (%rdi,%rdx),%rdi
loop loop_8
adcq %r9,%rax /* add in carry */
fold:
movl %eax,%ebx
shrq $32,%rax
addq %rbx,%rax
/* do last upto 6 bytes */
handle_7:
movl %r10d,%ecx
andl $7,%ecx
shrl $1,%ecx
jz handle_1
movl $2,%edx
xorl %ebx,%ebx
clc
loop_1:
source
movw (%rdi),%bx
adcq %rbx,%rax
dest
movw %bx,(%rsi)
addq %rdx,%rdi
addq %rdx,%rsi
loop loop_1
adcw %r9w,%ax /* add in carry */
/* handle last odd byte */
handle_1:
testl $1,%r10d
jz ende
xorl %ebx,%ebx
source
movb (%rdi),%bl
dest
movb %bl,(%rsi)
addw %bx,%ax
adcw %r9w,%ax /* carry */
ende:
sfence
popq %r15
popq %r14
popq %r12
popq %rbx
ret
#ifdef FIX_ALIGNMENT
/* align source to 8 bytes. */
/* r8d: unalignedness, ecx len */
bad_alignment:
testl $1,%edi
jnz odd_source
/* compute distance to next aligned position */
movl $8,%r8d
xchgl %r8d,%ecx
subl %r8d,%ecx
/* handle unaligned part */
shrl $1,%ecx
xorl %ebx,%ebx
movl $2,%r10d
align_loop:
source
movw (%rdi),%bx
addq %rbx,%rax /* carry cannot happen */
dest
movw %bx,(%rsi)
addq %r10,%rdi
addq %r10,%rsi
loop align_loop
jmp after_bad_alignment
/* weird case. need to swap the sum at the end because the spec requires
16 bit words of the sum to be always paired.
handle it recursively because it should be rather rare. */
odd_source:
/* copy odd byte */
xorl %ebx,%ebx
source
movb (%rdi),%bl
addl %ebx,%eax /* add to old checksum */
adcl $0,%ecx
dest
movb %al,(%rsi)
/* fix arguments */
movl %eax,%ecx
incq %rsi
incq %rdi
decq %rdx
call csum_partial_copy_generic
bswap %eax /* this should work, but check */
jmp ende
#endif
/* Exception handlers. Very simple, zeroing is done in the wrappers */
bad_source:
movl $-EFAULT,(%r14)
jmp ende
bad_dest:
movl $-EFAULT,(%r15)
jmp ende
arch/x86_64/lib/generic-checksum.c → arch/x86_64/lib/csum-partial.c
View file @ e4b9e2aa
/*
* arch/x86_64/lib/checksum.c
* arch/x86_64/lib/csum-partial.c
*
* This file contains network checksum routines that are better done
* in an architecture-specific manner due to speed..
* in an architecture-specific manner due to speed.
*/
#include <linux/string.h>
#include <asm/byteorder.h>
#include <linux/compiler.h>
#include <linux/module.h>
/* Better way for this sought */
static inline unsigned short from64to16(unsigned long x)
{
/* add up 32-bit words for 33 bits */
......@@ -22,13 +23,13 @@ static inline unsigned short from64to16(unsigned long x)
}
/*
* Do a 64-bit checksum on an arbitrary memory area..
* Do a 64-bit checksum on an arbitrary memory area.
* Returns a 32bit checksum.
*
* This isn't a great routine, but it's not _horrible_ either. The
* inner loop could be unrolled a bit further, and there are better
* ways to do the carry, but this is reasonable.
* This isn't a great routine, but it's not _horrible_ either.
* We rely on the compiler to unroll.
*/
static inline unsigned long do_csum(const unsigned char * buff, int len)
static inline unsigned do_csum(const unsigned char * buff, int len)
{
int odd, count;
unsigned long result = 0;
......@@ -36,7 +37,7 @@ static inline unsigned long do_csum(const unsigned char * buff, int len)
if (len <= 0)
goto out;
odd = 1 & (unsigned long) buff;
if (odd) {
if (unlikely(odd)) {
result = *buff << 8;
len--;
buff++;
......@@ -59,16 +60,15 @@ static inline unsigned long do_csum(const unsigned char * buff, int len)
}
count >>= 1; /* nr of 64-bit words.. */
if (count) {
unsigned long carry = 0;
unsigned long zero = 0;
do {
unsigned long w = *(unsigned long *) buff;
asm(" addq %1,%0\n"
" adcq %2,%0\n"
: "=r" (result)
: "m" (*buff), "r" (zero), "0" (result));
count--;
buff += 8;
result += carry;
result += w;
carry = (w > result);
} while (count);
result += carry;
result = (result & 0xffffffff) + (result >> 32);
}
if (len & 4) {
......@@ -84,8 +84,8 @@ static inline unsigned long do_csum(const unsigned char * buff, int len)
if (len & 1)
result += *buff;
result = from64to16(result);
if (odd)
result = ((result >> 8) & 0xff) | ((result & 0xff) << 8);
if (unlikely(odd))
return ((result >> 8) & 0xff) | ((result & 0xff) << 8);
out:
return result;
}
......@@ -100,25 +100,27 @@ static inline unsigned long do_csum(const unsigned char * buff, int len)
* this function must be called with even lengths, except
* for the last fragment, which may be odd
*
* it's best to have buff aligned on a 32-bit boundary
* it's best to have buff aligned on a 64-bit boundary
*/
unsigned int csum_partial(const unsigned char * buff, int len, unsigned int sum)
{
unsigned long result = do_csum(buff, len);
unsigned result = do_csum(buff, len);
/* add in old sum, and carry.. */
result += sum;
/* 32+c bits -> 32 bits */
result = (result & 0xffffffff) + (result >> 32);
asm("addl %1,%0\n\t"
"adcl $0,%0" : "=r" (result) : "r" (sum), "0" (result));
return result;
}
//EXPORT_SYMBOL(csum_partial);
/*
* this routine is used for miscellaneous IP-like checksums, mainly
* in icmp.c
*/
unsigned short ip_compute_csum(unsigned char * buff, int len)
{
return ~from64to16(do_csum(buff,len));
return ~csum_partial(buff,len,0);
}
EXPORT_SYMBOL(ip_compute_csum);
arch/x86_64/lib/csum-wrappers.c 0 → 100644
View file @ e4b9e2aa
/* Copyright 2002 Andi Kleen, SuSE Labs.
* Subject to the GNU Public License v.2
*
* Wrappers of assembly checksum functions for x86-64.
*/
#include <asm/checksum.h>
#include <linux/module.h>
/* Better way for this sought */
static inline unsigned from64to32(unsigned long x)
{
/* add up 32-bit words for 33 bits */
x = (x & 0xffffffff) + (x >> 32);
/* add up 16-bit and 17-bit words for 17+c bits */
x = (x & 0xffff) + (x >> 16);
/* add up 16-bit and 2-bit for 16+c bit */
x = (x & 0xffff) + (x >> 16);
return x;
}
/**
* csum_partial_copy_from_user - Copy and checksum from user space.
* @src: source address (user space)
* @dst: destination address
* @len: number of bytes to be copied.
* @isum: initial sum that is added into the result (32bit unfolded)
* @errp: set to -EFAULT for an bad source address.
*
* Returns an 32bit unfolded checksum of the buffer.
* src and dst are best aligned to 64bits.
*/
unsigned int
csum_partial_copy_from_user(const char *src, char *dst,
int len, unsigned int isum, int *errp)
{
*errp = 0;
if (likely(access_ok(VERIFY_READ,src, len))) {
unsigned long sum;
sum = csum_partial_copy_generic(src,dst,len,isum,errp,NULL);
if (likely(*errp == 0))
return from64to32(sum);
}
*errp = -EFAULT;
memset(dst,0,len);
return 0;
}
EXPORT_SYMBOL(csum_partial_copy_from_user);
/**
* csum_partial_copy_to_user - Copy and checksum to user space.
* @src: source address
* @dst: destination address (user space)
* @len: number of bytes to be copied.
* @isum: initial sum that is added into the result (32bit unfolded)
* @errp: set to -EFAULT for an bad destination address.
*
* Returns an 32bit unfolded checksum of the buffer.
* src and dst are best aligned to 64bits.
*/
unsigned int
csum_partial_copy_to_user(const char *src, char *dst,
int len, unsigned int isum, int *errp)
{
if (unlikely(!access_ok(VERIFY_WRITE, dst, len))) {
*errp = -EFAULT;
return 0;
}
*errp = 0;
return from64to32(csum_partial_copy_generic(src,dst,len,isum,NULL,errp));
}
EXPORT_SYMBOL(csum_partial_copy_to_user);
/**
* csum_partial_copy_nocheck - Copy and checksum.
* @src: source address
* @dst: destination address
* @len: number of bytes to be copied.
* @isum: initial sum that is added into the result (32bit unfolded)
*
* Returns an 32bit unfolded checksum of the buffer.
*/
unsigned int
csum_partial_copy_nocheck(const char *src, char *dst, int len, unsigned int sum)
{
return from64to32(csum_partial_copy_generic(src,dst,len,sum,NULL,NULL));
}
//EXPORT_SYMBOL(csum_partial_copy_nocheck);
unsigned short csum_ipv6_magic(struct in6_addr *saddr, struct in6_addr *daddr,
__u32 len, unsigned short proto, unsigned int sum)
{
__u64 rest, sum64;
rest = (__u64)htonl(len) + (__u64)htons(proto) + (__u64)sum;
asm(" addq (%[saddr]),%[sum]\n"
" adcq 8(%[saddr]),%[sum]\n"
" adcq (%[daddr]),%[sum]\n"
" adcq 8(%[daddr]),%[sum]\n"
" adcq $0,%[sum]\n"
: [sum] "=r" (sum64)
: "[sum]" (rest),[saddr] "r" (saddr), [daddr] "r" (daddr));
return csum_fold(from64to32(sum64));
}
EXPORT_SYMBOL(csum_ipv6_magic);
arch/x86_64/lib/io.c 0 → 100644
View file @ e4b9e2aa
#include <linux/string.h>
#include <asm/io.h>
#include <linux/module.h>
void *memcpy_toio(void *dst,void*src,unsigned len)
{
return __inline_memcpy(__io_virt(dst),src,len);
}
void *memcpy_fromio(void *dst,void*src,unsigned len)
{
return __inline_memcpy(dst,__io_virt(src),len);
}
EXPORT_SYMBOL(memcpy_toio);
EXPORT_SYMBOL(memcpy_fromio);
arch/x86_64/lib/memcpy.S 0 → 100644
View file @ e4b9e2aa
/* Copyright 2002 Andi Kleen */
/*
* memcpy - Copy a memory block.
*
* Input:
* rdi destination
* rsi source
* rdx count
*
* Output:
* rax original destination
*/
.globl __memcpy
.globl memcpy
.p2align
__memcpy:
memcpy:
pushq %rbx
prefetcht0 (%rsi) /*for more hopefully the hw prefetch will kick in*/
movq %rdi,%rax
movl %edi,%ecx
andl $7,%ecx
jnz bad_alignment
after_bad_alignment:
movq %rdx,%rcx
movl $64,%ebx
shrq $6,%rcx
jz handle_tail
loop_64:
/* no prefetch because we assume the hw prefetcher does it already
and we have no specific temporal hint to give. XXX or give a nta
hint for the source? */
movq (%rsi),%r11
movq 8(%rsi),%r8
movq 2*8(%rsi),%r9
movq 3*8(%rsi),%r10
movnti %r11,(%rdi)
movnti %r8,1*8(%rdi)
movnti %r9,2*8(%rdi)
movnti %r10,3*8(%rdi)
movq 4*8(%rsi),%r11
movq 5*8(%rsi),%r8
movq 6*8(%rsi),%r9
movq 7*8(%rsi),%r10
movnti %r11,4*8(%rdi)
movnti %r8,5*8(%rdi)
movnti %r9,6*8(%rdi)
movnti %r10,7*8(%rdi)
addq %rbx,%rsi
addq %rbx,%rdi
loop loop_64
handle_tail:
movl %edx,%ecx
andl $63,%ecx
shrl $3,%ecx
jz handle_7
movl $8,%ebx
loop_8:
movq (%rsi),%r8
movnti %r8,(%rdi)
addq %rbx,%rdi
addq %rbx,%rsi
loop loop_8
handle_7:
movl %edx,%ecx
andl $7,%ecx
jz ende
loop_1:
movb (%rsi),%r8b
movb %r8b,(%rdi)
incq %rdi
incq %rsi
loop loop_1
ende:
sfence
popq %rbx
ret
/* align destination */
/* This is simpleminded. For bigger blocks it may make sense to align
src and dst to their aligned subset and handle the rest separately */
bad_alignment:
movl $8,%r9d
subl %ecx,%r9d
movl %r9d,%ecx
subq %r9,%rdx
js small_alignment
jz small_alignment
align_1:
movb (%rsi),%r8b
movb %r8b,(%rdi)
incq %rdi
incq %rsi
loop align_1
jmp after_bad_alignment
small_alignment:
addq %r9,%rdx
jmp handle_7
arch/x86_64/lib/memmove.c 0 → 100644
View file @ e4b9e2aa
/* Normally compiler builtins are used, but sometimes the compiler calls out
of line code. Based on asm-i386/string.h.
*/
#define _STRING_C
#include <linux/string.h>
#undef memmove
void *memmove(void * dest,const void *src,size_t count)
{
if (dest < src) {
__inline_memcpy(dest,src,count);
} else {
/* Could be more clever and move longs */
unsigned long d0, d1, d2;
__asm__ __volatile__(
"std\n\t"
"rep\n\t"
"movsb\n\t"
"cld"
: "=&c" (d0), "=&S" (d1), "=&D" (d2)
:"0" (count),
"1" (count-1+(const char *)src),
"2" (count-1+(char *)dest)
:"memory");
}
return dest;
}
arch/x86_64/lib/memset.S 0 → 100644
View file @ e4b9e2aa
/* Copyright 2002 Andi Kleen, SuSE Labs */
// #define FIX_ALIGNMENT 1
/*
* ISO C memset - set a memory block to a byte value.
*
* rdi destination
* rsi value (char)
* rdx count (bytes)
*
* rax original destination
*/
.globl ____memset
.p2align
____memset:
movq %rdi,%r10 /* save destination for return address */
movq %rdx,%r11 /* save count */
/* expand byte value */
movzbl %sil,%ecx /* zero extend char value */
movabs $0x0101010101010101,%rax /* expansion pattern */
mul %rcx /* expand with rax, clobbers rdx */
#ifdef FIX_ALIGNMENT
/* align dst */
movl %edi,%r9d
andl $7,%r9d /* test unaligned bits */
jnz bad_alignment
after_bad_alignment:
#endif
movq %r11,%rcx /* restore count */
shrq $6,%rcx /* divide by 64 */
jz handle_tail /* block smaller than 64 bytes? */
movl $64,%r8d /* CSE loop block size */
loop_64:
movnti %rax,0*8(%rdi)
movnti %rax,1*8(%rdi)
movnti %rax,2*8(%rdi)
movnti %rax,3*8(%rdi)
movnti %rax,4*8(%rdi)
movnti %rax,5*8(%rdi)
movnti %rax,6*8(%rdi)
movnti %rax,7*8(%rdi) /* clear 64 byte blocks */
addq %r8,%rdi /* increase pointer by 64 bytes */
loop loop_64 /* decrement rcx and if not zero loop */
/* Handle tail in loops. The loops should be faster than hard
to predict jump tables. */
handle_tail:
movl %r11d,%ecx
andl $63,%ecx
shrl $3,%ecx
jz handle_7
loop_8:
movnti %rax,(%rdi) /* long words */
addq $8,%rdi
loop loop_8
handle_7:
movl %r11d,%ecx
andl $7,%ecx
jz ende
loop_1:
movb %al,(%rdi) /* bytes */
incq %rdi
loop loop_1
ende:
movq %r10,%rax
ret
#ifdef FIX_ALIGNMENT
bad_alignment:
andq $-8,%r11 /* shorter than 8 bytes */
jz handle_7 /* if yes handle it in the tail code */
movnti %rax,(%rdi) /* unaligned store of 8 bytes */
movq $8,%r8
subq %r9,%r8 /* compute alignment (8-misalignment) */
addq %r8,%rdi /* fix destination */
subq %r8,%r11 /* fix count */
jmp after_bad_alignment
#endif
arch/x86_64/lib/mmx.c deleted 100644 → 0
View file @ cc9af0c5
#include <linux/types.h>
#include <linux/string.h>
#include <linux/sched.h>
#include <linux/compiler.h>
#include <asm/i387.h>
#include <asm/hardirq.h>
#include <asm/page.h>
/*
* MMX 3DNow! library helper functions
*
* To do:
* We can use MMX just for prefetch in IRQ's. This may be a win.
* (reported so on K6-III)
* We should use a better code neutral filler for the short jump
* leal ebx. [ebx] is apparently best for K6-2, but Cyrix ??
* We also want to clobber the filler register so we dont get any
* register forwarding stalls on the filler.
*
* Add *user handling. Checksums are not a win with MMX on any CPU
* tested so far for any MMX solution figured.
*
* 22/09/2000 - Arjan van de Ven
* Improved for non-egineering-sample Athlons
*
* 2002 Andi Kleen. Some cleanups and changes for x86-64.
* Not really tuned yet. Using the Athlon version for now.
* This currenly uses MMX for 8 byte stores, but on hammer we could
* use integer 8 byte stores too and avoid the FPU save overhead.
* Disadvantage is that the integer load/stores have strong ordering
* model and may be slower.
*
* $Id$
*/
#ifdef MMX_MEMCPY_THRESH
void *_mmx_memcpy(void *to, const void *from, size_t len)
{
void *p;
int i;
p = to;
if (unlikely(in_interrupt()))
goto standard;
/* XXX: check if this is still memory bound with unaligned to/from.
if not align them here to 8bytes. */
i = len >> 6; /* len/64 */
kernel_fpu_begin();
__asm__ __volatile__ (
" prefetch (%0)\n" /* This set is 28 bytes */
" prefetch 64(%0)\n"
" prefetch 128(%0)\n"
" prefetch 192(%0)\n"
" prefetch 256(%0)\n"
"\n"
: : "r" (from) );
for(; i>5; i--)
{
__asm__ __volatile__ (
" prefetch 320(%0)\n"
" movq (%0), %%mm0\n"
" movq 8(%0), %%mm1\n"
" movq 16(%0), %%mm2\n"
" movq 24(%0), %%mm3\n"
" movq %%mm0, (%1)\n"
" movq %%mm1, 8(%1)\n"
" movq %%mm2, 16(%1)\n"
" movq %%mm3, 24(%1)\n"
" movq 32(%0), %%mm0\n"
" movq 40(%0), %%mm1\n"
" movq 48(%0), %%mm2\n"
" movq 56(%0), %%mm3\n"
" movq %%mm0, 32(%1)\n"
" movq %%mm1, 40(%1)\n"
" movq %%mm2, 48(%1)\n"
" movq %%mm3, 56(%1)\n"
: : "r" (from), "r" (to) : "memory");
from+=64;
to+=64;
}
for(; i>0; i--)
{
__asm__ __volatile__ (
" movq (%0), %%mm0\n"
" movq 8(%0), %%mm1\n"
" movq 16(%0), %%mm2\n"
" movq 24(%0), %%mm3\n"
" movq %%mm0, (%1)\n"
" movq %%mm1, 8(%1)\n"
" movq %%mm2, 16(%1)\n"
" movq %%mm3, 24(%1)\n"
" movq 32(%0), %%mm0\n"
" movq 40(%0), %%mm1\n"
" movq 48(%0), %%mm2\n"
" movq 56(%0), %%mm3\n"
" movq %%mm0, 32(%1)\n"
" movq %%mm1, 40(%1)\n"
" movq %%mm2, 48(%1)\n"
" movq %%mm3, 56(%1)\n"
: : "r" (from), "r" (to) : "memory");
from+=64;
to+=64;
}
len &= 63;
kernel_fpu_end();
/*
* Now do the tail of the block
*/
standard:
__inline_memcpy(to, from, len);
return p;
}
#endif
static inline void fast_clear_page(void *page)
{
int i;
kernel_fpu_begin();
__asm__ __volatile__ (
" pxor %%mm0, %%mm0\n" : :
);
for(i=0;i<4096/64;i++)
{
__asm__ __volatile__ (
" movntq %%mm0, (%0)\n"
" movntq %%mm0, 8(%0)\n"
" movntq %%mm0, 16(%0)\n"
" movntq %%mm0, 24(%0)\n"
" movntq %%mm0, 32(%0)\n"
" movntq %%mm0, 40(%0)\n"
" movntq %%mm0, 48(%0)\n"
" movntq %%mm0, 56(%0)\n"
: : "r" (page) : "memory");
page+=64;
}
/* since movntq is weakly-ordered, a "sfence" is needed to become
* ordered again.
*/
__asm__ __volatile__ (
" sfence \n" : :
);
kernel_fpu_end();
}
static inline void fast_copy_page(void *to, void *from)
{
int i;
kernel_fpu_begin();
/* maybe the prefetch stuff can go before the expensive fnsave...
* but that is for later. -AV
*/
__asm__ __volatile__ (
" prefetch (%0)\n"
" prefetch 64(%0)\n"
" prefetch 128(%0)\n"
" prefetch 192(%0)\n"
" prefetch 256(%0)\n"
: : "r" (from) );
for(i=0; i<(4096-320)/64; i++)
{
__asm__ __volatile__ (
" prefetch 320(%0)\n"
" movq (%0), %%mm0\n"
" movntq %%mm0, (%1)\n"
" movq 8(%0), %%mm1\n"
" movntq %%mm1, 8(%1)\n"
" movq 16(%0), %%mm2\n"
" movntq %%mm2, 16(%1)\n"
" movq 24(%0), %%mm3\n"
" movntq %%mm3, 24(%1)\n"
" movq 32(%0), %%mm4\n"
" movntq %%mm4, 32(%1)\n"
" movq 40(%0), %%mm5\n"
" movntq %%mm5, 40(%1)\n"
" movq 48(%0), %%mm6\n"
" movntq %%mm6, 48(%1)\n"
" movq 56(%0), %%mm7\n"
" movntq %%mm7, 56(%1)\n"
: : "r" (from), "r" (to) : "memory");
from+=64;
to+=64;
}
for(i=(4096-320)/64; i<4096/64; i++)
{
__asm__ __volatile__ (
"2: movq (%0), %%mm0\n"
" movntq %%mm0, (%1)\n"
" movq 8(%0), %%mm1\n"
" movntq %%mm1, 8(%1)\n"
" movq 16(%0), %%mm2\n"
" movntq %%mm2, 16(%1)\n"
" movq 24(%0), %%mm3\n"
" movntq %%mm3, 24(%1)\n"
" movq 32(%0), %%mm4\n"
" movntq %%mm4, 32(%1)\n"
" movq 40(%0), %%mm5\n"
" movntq %%mm5, 40(%1)\n"
" movq 48(%0), %%mm6\n"
" movntq %%mm6, 48(%1)\n"
" movq 56(%0), %%mm7\n"
" movntq %%mm7, 56(%1)\n"
: : "r" (from), "r" (to) : "memory");
from+=64;
to+=64;
}
/* since movntq is weakly-ordered, a "sfence" is needed to become
* ordered again.
*/
__asm__ __volatile__ (
" sfence \n" : :
);
kernel_fpu_end();
}
void mmx_clear_page(void * page)
{
#if 1
__builtin_memset(page,0,PAGE_SIZE);
#else
/* AK: these in_interrupt checks should not be needed. */
if(unlikely(in_interrupt()))
__builtin_memset(page,0,PAGE_SIZE);
else
fast_clear_page(page);
#endif
}
void mmx_copy_page(void *to, void *from)
{
#if 1
__builtin_memcpy(to,from,PAGE_SIZE);
#else
/* AK: these in_interrupt checks should not be needed. */
if(unlikely(in_interrupt()))
__builtin_memcpy(to,from,PAGE_SIZE);
else
fast_copy_page(to, from);
#endif
}
arch/x86_64/lib/rwsem_thunk.S deleted 100644 → 0
View file @ cc9af0c5
#include <asm/calling.h>
/*
* Save registers for the slow path of semaphores here to avoid
* disturbance of register allocation in fast paths with function calls.
* Written 2001 by Andi Kleen.
*/
.macro rwsem_thunk name,func
.globl \name
\name:
SAVE_ARGS
movq %rax,%rdi
call \func
jmp restore
.endm
rwsem_thunk rwsem_down_read_failed_thunk,rwsem_down_read_failed
rwsem_thunk rwsem_down_write_failed_thunk,rwsem_down_write_failed
rwsem_thunk rwsem_wake_thunk,rwsem_wake
/* This does not really belong here, but the macros are so
convenient. */
rwsem_thunk do_softirq_thunk,do_softirq
restore:
RESTORE_ARGS
ret
arch/x86_64/lib/usercopy.c
View file @ e4b9e2aa
/*
* User address space access functions.
* The non inlined parts of asm-i386/uaccess.h are here.
*
* Copyright 1997 Andi Kleen <ak@muc.de>
* Copyright 1997 Linus Torvalds
* Copyright 2002 Andi Kleen <ak@suse.de>
*/
#include <linux/config.h>
#include <asm/uaccess.h>
#include <asm/mmx.h>
unsigned long
__generic_copy_to_user(void *to, const void *from, unsigned long n)
{
prefetch(from);
if (access_ok(VERIFY_WRITE, to, n))
__copy_user(to,from,n);
return n;
}
unsigned long
__generic_copy_from_user(void *to, const void *from, unsigned long n)
{
prefetchw(to);
if (access_ok(VERIFY_READ, from, n))
__copy_user_zeroing(to,from,n);
else
memset(to, 0, n);
return n;
}
/*
* Copy a null terminated string from userspace.
*/
#define __do_strncpy_from_user(dst,src,count,res) \
do { \
long __d0, __d1, __d2; \
__asm__ __volatile__( \
" testq %1,%1\n" \
" jz 2f\n" \
"0: lodsb\n" \
" stosb\n" \
" testb %%al,%%al\n" \
" jz 1f\n" \
" decq %1\n" \
" jnz 0b\n" \
"1: subq %1,%0\n" \
"2:\n" \
".section .fixup,\"ax\"\n" \
"3: movq %5,%0\n" \
" jmp 2b\n" \
".previous\n" \
".section __ex_table,\"a\"\n" \
" .align 4\n" \
" .quad 0b,3b\n" \
".previous" \
: "=r"(res), "=c"(count), "=&a" (__d0), "=&S" (__d1), \
"=&D" (__d2) \
: "i"(-EFAULT), "0"(count), "1"(count), "3"(src), "4"(dst) \
: "memory"); \
} while (0)
long
__strncpy_from_user(char *dst, const char *src, long count)
long __strncpy_from_user(char *dst, const char *src, long count)
{
long res;
__do_strncpy_from_user(dst, src, count, res);
long __d0, __d1, __d2;
asm volatile( \
" testq %1,%1\n"
" jz 2f\n"
"0: lodsb\n"
" stosb\n"
" testb %%al,%%al\n"
" loopnz 0b\n"
"1: subq %1,%0\n"
"2:\n"
".section .fixup,\"ax\"\n"
"3: movq %5,%0\n"
" jmp 2b\n"
".previous\n"
".section __ex_table,\"a\"\n"
" .align 8\n"
" .quad 0b,3b\n"
".previous"
: "=r"(res), "=c"(count), "=&a" (__d0), "=&S" (__d1),
"=&D" (__d2)
: "i"(-EFAULT), "0"(count), "1"(count), "3"(src), "4"(dst)
: "memory");
return res;
}
long
strncpy_from_user(char *dst, const char *src, long count)
long strncpy_from_user(char *dst, const char *src, long count)
{
long res = -EFAULT;
if (access_ok(VERIFY_READ, src, 1))
__do_strncpy_from_user(dst, src, count, res);
return res;
return __strncpy_from_user(dst, src, count);
return -EFAULT;
}
/*
* Zero Userspace
*/
#define __do_clear_user(addr,size) \
do { \
long __d0; \
__asm__ __volatile__( \
"cld\n" \
"0: rep; stosl\n" \
" movq %2,%0\n" \
"1: rep; stosb\n" \
"2:\n" \
".section .fixup,\"ax\"\n" \
"3: lea 0(%2,%0,4),%0\n" \
" jmp 2b\n" \
".previous\n" \
".section __ex_table,\"a\"\n" \
" .align 4\n" \
" .quad 0b,3b\n" \
" .quad 1b,2b\n" \
".previous" \
: "=&c"(size), "=&D" (__d0) \
: "r"(size & 3), "0"(size / 4), "1"(addr), "a"(0)); \
} while (0)
unsigned long
clear_user(void *to, unsigned long n)
unsigned long __clear_user(void *addr, unsigned long size)
{
if (access_ok(VERIFY_WRITE, to, n))
__do_clear_user(to, n);
return n;
long __d0;
/* no memory constraint because it doesn't change any memory gcc knows
about */
asm volatile(
" testq %[size8],%[size8]\n"
" jz 4f\n"
"0: movnti %[zero],(%[dst])\n"
" addq %[eight],%[dst]\n"
" loop 0b\n"
"4: movq %[size1],%%rcx\n"
" testl %%ecx,%%ecx\n"
" jz 2f\n"
"1: movb %b[zero],(%[dst])\n"
" incq %[dst]\n"
" loop 1b\n"
"2: sfence\n"
".section .fixup,\"ax\"\n"
"3: lea 0(%[size1],%[size8],8),%[size8]\n"
" jmp 2b\n"
".previous\n"
".section __ex_table,\"a\"\n"
" .align 8\n"
" .quad 0b,3b\n"
" .quad 1b,2b\n"
".previous"
: [size8] "=c"(size), [dst] "=&D" (__d0)
: [size1] "r"(size & 7), "[size8]" (size / 8), "[dst] "(addr),
[zero] "r" (0UL), [eight] "r" (8UL));
return size;
}
unsigned long
__clear_user(void *to, unsigned long n)
unsigned long clear_user(void *to, unsigned long n)
{
__do_clear_user(to, n);
if (access_ok(VERIFY_WRITE, to, n))
return __clear_user(to, n);
return n;
}
......
arch/x86_64/mm/extable.c
View file @ e4b9e2aa
/*
* linux/arch/i386/mm/extable.c
* linux/arch/x86_64/mm/extable.c
*/
#include <linux/config.h>
#include <linux/module.h>
#include <linux/spinlock.h>
#include <asm/uaccess.h>
#include <linux/init.h>
extern const struct exception_table_entry __start___ex_table[];
extern const struct exception_table_entry __stop___ex_table[];
static inline unsigned long
void __init exception_table_check(void)
{
const struct exception_table_entry *e;
unsigned long prev;
prev = 0;
for (e = __start___ex_table; e < __stop___ex_table; e++) {
if (e->insn < prev) {
panic("unordered exception table at %016lx:%016lx and %016lx:%016lx\n",
prev, e[-1].fixup,
e->insn, e->fixup);
}
prev = e->insn;
}
}
static unsigned long
search_one_table(const struct exception_table_entry *first,
const struct exception_table_entry *last,
unsigned long value)
......@@ -41,8 +59,7 @@ search_exception_table(unsigned long addr)
#ifndef CONFIG_MODULES
/* There is only the kernel to search. */
ret = search_one_table(__start___ex_table, __stop___ex_table-1, addr);
return ret;
return search_one_table(__start___ex_table, __stop___ex_table-1, addr);
#else
/* The kernel is the last "module" -- no need to treat it special. */
struct module *mp;
......
arch/x86_64/mm/fault.c
View file @ e4b9e2aa
......@@ -85,7 +85,7 @@ void dump_pagetable(unsigned long address)
}
int page_fault_trace;
int exception_trace;
int exception_trace = 1;
/*
* This routine handles page faults. It determines the address,
......@@ -311,11 +311,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long error_code)
if (!pte_present(*pte))
goto bad_area_nosemaphore;
/* Strictly a flush_tlb_all because vmalloc is global,
but this only applies to new global pages so it should
not be needed. vmalloc will likely touch multiple ptes,
so do a full flush instead of a partial one. */
__flush_tlb();
__flush_tlb_all();
return;
}
}
arch/x86_64/mm/init.c
View file @ e4b9e2aa
......@@ -153,6 +153,7 @@ static struct temp_map {
} temp_mappings[] __initdata = {
{ &temp_boot_pmds[0], (void *)(40UL * 1024 * 1024) },
{ &temp_boot_pmds[1], (void *)(42UL * 1024 * 1024) },
{ &temp_boot_pmds[1], (void *)(44UL * 1024 * 1024) },
{}
};
......
arch/x86_64/mm/ioremap.c
View file @ e4b9e2aa
......@@ -9,6 +9,7 @@
*/
#include <linux/vmalloc.h>
#include <linux/init.h>
#include <asm/io.h>
#include <asm/pgalloc.h>
#include <asm/fixmap.h>
......@@ -165,5 +166,5 @@ void * __ioremap(unsigned long phys_addr, unsigned long size, unsigned long flag
void iounmap(void *addr)
{
if (addr > high_memory)
return vfree((void *) (PAGE_MASK & (unsigned long) addr));
vfree((void *) (PAGE_MASK & (unsigned long) addr));
}
arch/x86_64/pci/Makefile 0 → 100644
View file @ e4b9e2aa
O_TARGET := pci.o
obj-y := x86-64.o
obj-$(CONFIG_PCI_DIRECT) += direct.o
obj-y += fixup.o
ifdef CONFIG_ACPI_PCI
obj-y += acpi.o
endif
obj-y += legacy.o
obj-y += irq.o common.o
include $(TOPDIR)/Rules.make
arch/x86_64/pci/acpi.c 0 → 100644
View file @ e4b9e2aa
#include <linux/pci.h>
#include <linux/acpi.h>
#include <linux/init.h>
#include "pci.h"
static int __init pci_acpi_init(void)
{
if (pcibios_scanned)
return 0;
if (!(pci_probe & PCI_NO_ACPI_ROUTING)) {
if (!acpi_pci_irq_init()) {
printk(KERN_INFO "PCI: Using ACPI for IRQ routing\n");
printk(KERN_INFO "PCI: if you experience problems, try using option 'pci=noacpi'\n");
pcibios_scanned++;
pcibios_enable_irq = acpi_pci_irq_enable;
} else
printk(KERN_WARNING "PCI: Invalid ACPI-PCI IRQ routing table\n");
}
return 0;
}
subsys_initcall(pci_acpi_init);
arch/x86_64/pci/changelog 0 → 100644
View file @ e4b9e2aa
See arch/i386/pci/changelog for early changelog.
arch/x86_64/pci/common.c 0 → 100644
View file @ e4b9e2aa
/*
* Low-Level PCI Support for PC
*
* (c) 1999--2000 Martin Mares <mj@ucw.cz>
Note: on x86-64 there is no PCI BIOS so there is no way to sort in the
same order as 32bit Linux. This could cause grief for dualbooting because
devices may wander. May want to use ACPI for sorting eventually.
*/
#include <linux/sched.h>
#include <linux/pci.h>
#include <linux/init.h>
#include <linux/ioport.h>
#include <asm/segment.h>
#include <asm/io.h>
#include <asm/smp.h>
#include "pci.h"
unsigned int pci_probe = PCI_PROBE_CONF1 | PCI_PROBE_CONF2;
int pcibios_last_bus = 0xfe; /* XXX */
struct pci_bus *pci_root_bus = NULL;
struct pci_ops *pci_root_ops = NULL;
int (*pci_config_read)(int seg, int bus, int dev, int fn, int reg, int len, u32 *value) = NULL;
int (*pci_config_write)(int seg, int bus, int dev, int fn, int reg, int len, u32 value) = NULL;
/*
* legacy, numa, and acpi all want to call pcibios_scan_root
* from their initcalls. This flag prevents that.
*/
int pcibios_scanned;
/*
* This interrupt-safe spinlock protects all accesses to PCI
* configuration space.
*/
spinlock_t pci_config_lock = SPIN_LOCK_UNLOCKED;
/*
* Several buggy motherboards address only 16 devices and mirror
* them to next 16 IDs. We try to detect this `feature' on all
* primary buses (those containing host bridges as they are
* expected to be unique) and remove the ghost devices.
*/
static void __devinit pcibios_fixup_ghosts(struct pci_bus *b)
{
struct list_head *ln, *mn;
struct pci_dev *d, *e;
int mirror = PCI_DEVFN(16,0);
int seen_host_bridge = 0;
int i;
DBG("PCI: Scanning for ghost devices on bus %d\n", b->number);
for (ln=b->devices.next; ln != &b->devices; ln=ln->next) {
d = pci_dev_b(ln);
if ((d->class >> 8) == PCI_CLASS_BRIDGE_HOST)
seen_host_bridge++;
for (mn=ln->next; mn != &b->devices; mn=mn->next) {
e = pci_dev_b(mn);
if (e->devfn != d->devfn + mirror ||
e->vendor != d->vendor ||
e->device != d->device ||
e->class != d->class)
continue;
for(i=0; i<PCI_NUM_RESOURCES; i++)
if (e->resource[i].start != d->resource[i].start ||
e->resource[i].end != d->resource[i].end ||
e->resource[i].flags != d->resource[i].flags)
continue;
break;
}
if (mn == &b->devices)
return;
}
if (!seen_host_bridge)
return;
printk(KERN_WARNING "PCI: Ignoring ghost devices on bus %02x\n", b->number);
ln = &b->devices;
while (ln->next != &b->devices) {
d = pci_dev_b(ln->next);
if (d->devfn >= mirror) {
list_del(&d->global_list);
list_del(&d->bus_list);
kfree(d);
} else
ln = ln->next;
}
}
/*
* Called after each bus is probed, but before its children
* are examined.
*/
void __devinit pcibios_fixup_bus(struct pci_bus *b)
{
pcibios_fixup_ghosts(b);
pci_read_bridge_bases(b);
}
struct pci_bus * __devinit pcibios_scan_root(int busnum)
{
struct list_head *list;
struct pci_bus *bus;
list_for_each(list, &pci_root_buses) {
bus = pci_bus_b(list);
if (bus->number == busnum) {
/* Already scanned */
return bus;
}
}
printk("PCI: Probing PCI hardware (bus %02x)\n", busnum);
return pci_scan_bus(busnum, pci_root_ops, NULL);
}
static int __init pcibios_init(void)
{
if (!pci_root_ops) {
printk("PCI: System does not support PCI\n");
return 0;
}
pcibios_resource_survey();
/* may eventually need to do ACPI sort here. */
return 0;
}
subsys_initcall(pcibios_init);
char * __devinit pcibios_setup(char *str)
{
if (!strcmp(str, "off")) {
pci_probe = 0;
return NULL;
}
#ifdef CONFIG_PCI_DIRECT
else if (!strcmp(str, "conf1")) {
pci_probe = PCI_PROBE_CONF1 | PCI_NO_CHECKS;
return NULL;
}
else if (!strcmp(str, "conf2")) {
pci_probe = PCI_PROBE_CONF2 | PCI_NO_CHECKS;
return NULL;
}
#endif
#ifdef CONFIG_ACPI_PCI
else if (!strcmp(str, "noacpi")) {
pci_probe |= PCI_NO_ACPI_ROUTING;
return NULL;
}
#endif
else if (!strcmp(str, "rom")) {
pci_probe |= PCI_ASSIGN_ROMS;
return NULL;
} else if (!strcmp(str, "assign-busses")) {
pci_probe |= PCI_ASSIGN_ALL_BUSSES;
return NULL;
} else if (!strcmp(str, "usepirqmask")) {
pci_probe |= PCI_USE_PIRQ_MASK;
return NULL;
} else if (!strncmp(str, "irqmask=", 8)) {
pcibios_irq_mask = simple_strtol(str+8, NULL, 0);
return NULL;
} else if (!strncmp(str, "lastbus=", 8)) {
pcibios_last_bus = simple_strtol(str+8, NULL, 0);
return NULL;
}
return str;
}
unsigned int pcibios_assign_all_busses(void)
{
return (pci_probe & PCI_ASSIGN_ALL_BUSSES) ? 1 : 0;
}
int pcibios_enable_device(struct pci_dev *dev)
{
int err;
if ((err = pcibios_enable_resources(dev)) < 0)
return err;
return pcibios_enable_irq(dev);
}
arch/x86_64/kernel/pci-pc.c → arch/x86_64/pci/direct.c
View file @ e4b9e2aa
/*
* Low-Level PCI Support for PC
*
* (c) 1999--2000 Martin Mares <mj@ucw.cz>
* 2001 Andi Kleen. Cleanup for x86-64. Removed PCI-BIOS access and fixups
* for hardware that is unlikely to exist on any Hammer platform.
*
* On x86-64 we don't have any access to the PCI-BIOS in long mode, so we
* cannot sort the pci device table based on what the BIOS did. This might
* change the probing order of some devices compared to an i386 kernel.
* May need to use ACPI to fix this.
* direct.c - Low-level direct PCI config space access
*/
#include <linux/config.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/sched.h>
#include <linux/pci.h>
#include <linux/init.h>
#include <linux/ioport.h>
#include "pci.h"
#include <asm/segment.h>
#include <asm/io.h>
/*
* Functions for accessing PCI configuration space with type 1 accesses
*/
#include "pci-x86_64.h"
#define PCI_CONF1_ADDRESS(bus, dev, fn, reg) \
(0x80000000 | (bus << 16) | (dev << 11) | (fn << 8) | (reg & ~3))
unsigned int pci_probe = PCI_PROBE_BIOS | PCI_PROBE_CONF1 | PCI_PROBE_CONF2;
static int pci_conf1_read (int seg, int bus, int dev, int fn, int reg, int len, u32 *value)
{
unsigned long flags;
int pcibios_last_bus = -1;
struct pci_bus *pci_root_bus;
struct pci_ops *pci_root_ops;
if (!value || (bus > 255) || (dev > 31) || (fn > 7) || (reg > 255))
return -EINVAL;
/*
* Direct access to PCI hardware...
*/
spin_lock_irqsave(&pci_config_lock, flags);
#ifdef CONFIG_PCI_DIRECT
outl(PCI_CONF1_ADDRESS(bus, dev, fn, reg), 0xCF8);
/*
* Functions for accessing PCI configuration space with type 1 accesses
*/
switch (len) {
case 1:
*value = inb(0xCFC + (reg & 3));
break;
case 2:
*value = inw(0xCFC + (reg & 2));
break;
case 4:
*value = inl(0xCFC);
break;
}
spin_unlock_irqrestore(&pci_config_lock, flags);
return 0;
}
#define CONFIG_CMD(dev, where) (0x80000000 | (dev->bus->number << 16) | (dev->devfn << 8) | (where & ~3))
static int pci_conf1_write (int seg, int bus, int dev, int fn, int reg, int len, u32 value)
{
unsigned long flags;
if ((bus > 255) || (dev > 31) || (fn > 7) || (reg > 255))
return -EINVAL;
spin_lock_irqsave(&pci_config_lock, flags);
outl(PCI_CONF1_ADDRESS(bus, dev, fn, reg), 0xCF8);
switch (len) {
case 1:
outb((u8)value, 0xCFC + (reg & 3));
break;
case 2:
outw((u16)value, 0xCFC + (reg & 2));
break;
case 4:
outl((u32)value, 0xCFC);
break;
}
spin_unlock_irqrestore(&pci_config_lock, flags);
return 0;
}
#undef PCI_CONF1_ADDRESS
static int pci_conf1_read_config_byte(struct pci_dev *dev, int where, u8 *value)
{
outl(CONFIG_CMD(dev,where), 0xCF8);
*value = inb(0xCFC + (where&3));
return PCIBIOS_SUCCESSFUL;
int result;
u32 data;
if (!value)
return -EINVAL;
result = pci_conf1_read(0, dev->bus->number, PCI_SLOT(dev->devfn),
PCI_FUNC(dev->devfn), where, 1, &data);
*value = (u8)data;
return result;
}
static int pci_conf1_read_config_word(struct pci_dev *dev, int where, u16 *value)
{
outl(CONFIG_CMD(dev,where), 0xCF8);
*value = inw(0xCFC + (where&2));
return PCIBIOS_SUCCESSFUL;
int result;
u32 data;
if (!value)
return -EINVAL;
result = pci_conf1_read(0, dev->bus->number, PCI_SLOT(dev->devfn),
PCI_FUNC(dev->devfn), where, 2, &data);
*value = (u16)data;
return result;
}
static int pci_conf1_read_config_dword(struct pci_dev *dev, int where, u32 *value)
{
outl(CONFIG_CMD(dev,where), 0xCF8);
*value = inl(0xCFC);
return PCIBIOS_SUCCESSFUL;
if (!value)
return -EINVAL;
return pci_conf1_read(0, dev->bus->number, PCI_SLOT(dev->devfn),
PCI_FUNC(dev->devfn), where, 4, value);
}
static int pci_conf1_write_config_byte(struct pci_dev *dev, int where, u8 value)
{
outl(CONFIG_CMD(dev,where), 0xCF8);
outb(value, 0xCFC + (where&3));
return PCIBIOS_SUCCESSFUL;
return pci_conf1_write(0, dev->bus->number, PCI_SLOT(dev->devfn),
PCI_FUNC(dev->devfn), where, 1, value);
}
static int pci_conf1_write_config_word(struct pci_dev *dev, int where, u16 value)
{
outl(CONFIG_CMD(dev,where), 0xCF8);
outw(value, 0xCFC + (where&2));
return PCIBIOS_SUCCESSFUL;
return pci_conf1_write(0, dev->bus->number, PCI_SLOT(dev->devfn),
PCI_FUNC(dev->devfn), where, 2, value);
}
static int pci_conf1_write_config_dword(struct pci_dev *dev, int where, u32 value)
{
outl(CONFIG_CMD(dev,where), 0xCF8);
outl(value, 0xCFC);
return PCIBIOS_SUCCESSFUL;
return pci_conf1_write(0, dev->bus->number, PCI_SLOT(dev->devfn),
PCI_FUNC(dev->devfn), where, 4, value);
}
#undef CONFIG_CMD
static struct pci_ops pci_direct_conf1 = {
pci_conf1_read_config_byte,
pci_conf1_read_config_word,
......@@ -95,68 +140,127 @@ static struct pci_ops pci_direct_conf1 = {
pci_conf1_write_config_dword
};
/*
* Functions for accessing PCI configuration space with type 2 accesses
*/
#define IOADDR(devfn, where) ((0xC000 | ((devfn & 0x78) << 5)) + where)
#define FUNC(devfn) (((devfn & 7) << 1) | 0xf0)
#define SET(dev) if (dev->devfn & 0x80) return PCIBIOS_DEVICE_NOT_FOUND; \
outb(FUNC(dev->devfn), 0xCF8); \
outb(dev->bus->number, 0xCFA);
#define PCI_CONF2_ADDRESS(dev, reg) (u16)(0xC000 | (dev << 8) | reg)
static int pci_conf2_read_config_byte(struct pci_dev *dev, int where, u8 *value)
static int pci_conf2_read (int seg, int bus, int dev, int fn, int reg, int len, u32 *value)
{
SET(dev);
*value = inb(IOADDR(dev->devfn,where));
unsigned long flags;
if (!value || (bus > 255) || (dev > 31) || (fn > 7) || (reg > 255))
return -EINVAL;
if (dev & 0x10)
return PCIBIOS_DEVICE_NOT_FOUND;
spin_lock_irqsave(&pci_config_lock, flags);
outb((u8)(0xF0 | (fn << 1)), 0xCF8);
outb((u8)bus, 0xCFA);
switch (len) {
case 1:
*value = inb(PCI_CONF2_ADDRESS(dev, reg));
break;
case 2:
*value = inw(PCI_CONF2_ADDRESS(dev, reg));
break;
case 4:
*value = inl(PCI_CONF2_ADDRESS(dev, reg));
break;
}
outb (0, 0xCF8);
return PCIBIOS_SUCCESSFUL;
spin_unlock_irqrestore(&pci_config_lock, flags);
return 0;
}
static int pci_conf2_write (int seg, int bus, int dev, int fn, int reg, int len, u32 value)
{
unsigned long flags;
if ((bus > 255) || (dev > 31) || (fn > 7) || (reg > 255))
return -EINVAL;
if (dev & 0x10)
return PCIBIOS_DEVICE_NOT_FOUND;
spin_lock_irqsave(&pci_config_lock, flags);
outb((u8)(0xF0 | (fn << 1)), 0xCF8);
outb((u8)bus, 0xCFA);
switch (len) {
case 1:
outb ((u8)value, PCI_CONF2_ADDRESS(dev, reg));
break;
case 2:
outw ((u16)value, PCI_CONF2_ADDRESS(dev, reg));
break;
case 4:
outl ((u32)value, PCI_CONF2_ADDRESS(dev, reg));
break;
}
outb (0, 0xCF8);
spin_unlock_irqrestore(&pci_config_lock, flags);
return 0;
}
#undef PCI_CONF2_ADDRESS
static int pci_conf2_read_config_byte(struct pci_dev *dev, int where, u8 *value)
{
int result;
u32 data;
result = pci_conf2_read(0, dev->bus->number, PCI_SLOT(dev->devfn),
PCI_FUNC(dev->devfn), where, 1, &data);
*value = (u8)data;
return result;
}
static int pci_conf2_read_config_word(struct pci_dev *dev, int where, u16 *value)
{
SET(dev);
*value = inw(IOADDR(dev->devfn,where));
outb (0, 0xCF8);
return PCIBIOS_SUCCESSFUL;
int result;
u32 data;
result = pci_conf2_read(0, dev->bus->number, PCI_SLOT(dev->devfn),
PCI_FUNC(dev->devfn), where, 2, &data);
*value = (u16)data;
return result;
}
static int pci_conf2_read_config_dword(struct pci_dev *dev, int where, u32 *value)
{
SET(dev);
*value = inl (IOADDR(dev->devfn,where));
outb (0, 0xCF8);
return PCIBIOS_SUCCESSFUL;
return pci_conf2_read(0, dev->bus->number, PCI_SLOT(dev->devfn),
PCI_FUNC(dev->devfn), where, 4, value);
}
static int pci_conf2_write_config_byte(struct pci_dev *dev, int where, u8 value)
{
SET(dev);
outb (value, IOADDR(dev->devfn,where));
outb (0, 0xCF8);
return PCIBIOS_SUCCESSFUL;
return pci_conf2_write(0, dev->bus->number, PCI_SLOT(dev->devfn),
PCI_FUNC(dev->devfn), where, 1, value);
}
static int pci_conf2_write_config_word(struct pci_dev *dev, int where, u16 value)
{
SET(dev);
outw (value, IOADDR(dev->devfn,where));
outb (0, 0xCF8);
return PCIBIOS_SUCCESSFUL;
return pci_conf2_write(0, dev->bus->number, PCI_SLOT(dev->devfn),
PCI_FUNC(dev->devfn), where, 2, value);
}
static int pci_conf2_write_config_dword(struct pci_dev *dev, int where, u32 value)
{
SET(dev);
outl (value, IOADDR(dev->devfn,where));
outb (0, 0xCF8);
return PCIBIOS_SUCCESSFUL;
return pci_conf2_write(0, dev->bus->number, PCI_SLOT(dev->devfn),
PCI_FUNC(dev->devfn), where, 4, value);
}
#undef SET
#undef IOADDR
#undef FUNC
static struct pci_ops pci_direct_conf2 = {
pci_conf2_read_config_byte,
pci_conf2_read_config_word,
......@@ -166,6 +270,7 @@ static struct pci_ops pci_direct_conf2 = {
pci_conf2_write_config_dword
};
/*
* Before we decide to use direct hardware access mechanisms, we try to do some
* trivial checks to ensure it at least _seems_ to be working -- we just test
......@@ -214,8 +319,9 @@ static struct pci_ops * __devinit pci_check_direct(void)
pci_sanity_check(&pci_direct_conf1)) {
outl (tmp, 0xCF8);
__restore_flags(flags);
printk("PCI: Using configuration type 1\n");
request_region(0xCF8, 8, "PCI conf1");
printk(KERN_INFO "PCI: Using configuration type 1\n");
if (!request_region(0xCF8, 8, "PCI conf1"))
return NULL;
return &pci_direct_conf1;
}
outl (tmp, 0xCF8);
......@@ -231,8 +337,9 @@ static struct pci_ops * __devinit pci_check_direct(void)
if (inb (0xCF8) == 0x00 && inb (0xCFA) == 0x00 &&
pci_sanity_check(&pci_direct_conf2)) {
__restore_flags(flags);
printk("PCI: Using configuration type 2\n");
request_region(0xCF8, 4, "PCI conf2");
printk(KERN_INFO "PCI: Using configuration type 2\n");
if (!request_region(0xCF8, 4, "PCI conf2"))
return NULL;
return &pci_direct_conf2;
}
}
......@@ -241,207 +348,20 @@ static struct pci_ops * __devinit pci_check_direct(void)
return NULL;
}
#endif
/*
* Several buggy motherboards address only 16 devices and mirror
* them to next 16 IDs. We try to detect this `feature' on all
* primary buses (those containing host bridges as they are
* expected to be unique) and remove the ghost devices.
*/
static void __devinit pcibios_fixup_ghosts(struct pci_bus *b)
static int __init pci_direct_init(void)
{
struct list_head *ln, *mn;
struct pci_dev *d, *e;
int mirror = PCI_DEVFN(16,0);
int seen_host_bridge = 0;
int i;
DBG("PCI: Scanning for ghost devices on bus %d\n", b->number);
for (ln=b->devices.next; ln != &b->devices; ln=ln->next) {
d = pci_dev_b(ln);
if ((d->class >> 8) == PCI_CLASS_BRIDGE_HOST)
seen_host_bridge++;
for (mn=ln->next; mn != &b->devices; mn=mn->next) {
e = pci_dev_b(mn);
if (e->devfn != d->devfn + mirror ||
e->vendor != d->vendor ||
e->device != d->device ||
e->class != d->class)
continue;
for(i=0; i<PCI_NUM_RESOURCES; i++)
if (e->resource[i].start != d->resource[i].start ||
e->resource[i].end != d->resource[i].end ||
e->resource[i].flags != d->resource[i].flags)
continue;
break;
if ((pci_probe & (PCI_PROBE_CONF1 | PCI_PROBE_CONF2))
&& (pci_root_ops = pci_check_direct())) {
if (pci_root_ops == &pci_direct_conf1) {
pci_config_read = pci_conf1_read;
pci_config_write = pci_conf1_write;
}
else {
pci_config_read = pci_conf2_read;
pci_config_write = pci_conf2_write;
}
if (mn == &b->devices)
return;
}
if (!seen_host_bridge)
return;
printk("PCI: Ignoring ghost devices on bus %02x\n", b->number);
ln = &b->devices;
while (ln->next != &b->devices) {
d = pci_dev_b(ln->next);
if (d->devfn >= mirror) {
list_del(&d->global_list);
list_del(&d->bus_list);
kfree(d);
} else
ln = ln->next;
}
}
/*
* Discover remaining PCI buses in case there are peer host bridges.
* We use the number of last PCI bus provided by the PCI BIOS.
*/
static void __devinit pcibios_fixup_peer_bridges(void)
{
int n;
struct pci_bus bus;
struct pci_dev dev;
u16 l;
if (pcibios_last_bus <= 0 || pcibios_last_bus >= 0xff)
return;
DBG("PCI: Peer bridge fixup\n");
for (n=0; n <= pcibios_last_bus; n++) {
if (pci_bus_exists(&pci_root_buses, n))
continue;
bus.number = n;
bus.ops = pci_root_ops;
dev.bus = &bus;
for(dev.devfn=0; dev.devfn<256; dev.devfn += 8)
if (!pci_read_config_word(&dev, PCI_VENDOR_ID, &l) &&
l != 0x0000 && l != 0xffff) {
DBG("Found device at %02x:%02x [%04x]\n", n, dev.devfn, l);
printk("PCI: Discovered peer bus %02x\n", n);
pci_scan_bus(n, pci_root_ops, NULL);
break;
}
}
}
struct pci_fixup pcibios_fixups[] = {
/* Currently no fixup for hammer systems. May need to readd them
as needed. */
{ 0 }
};
/*
* Called after each bus is probed, but before its children
* are examined.
*/
void __devinit pcibios_fixup_bus(struct pci_bus *b)
{
pcibios_fixup_ghosts(b);
pci_read_bridge_bases(b);
}
/*
* Initialization. Try all known PCI access methods. Note that we support
* using both PCI BIOS and direct access: in such cases, we use I/O ports
* to access config space, but we still keep BIOS order of cards to be
* compatible with 2.0.X. This should go away some day.
*/
void __devinit pcibios_init(void)
{
struct pci_ops *bios = NULL;
struct pci_ops *dir = NULL;
#ifdef CONFIG_PCI_DIRECT
if (pci_probe & (PCI_PROBE_CONF1 | PCI_PROBE_CONF2))
dir = pci_check_direct();
#endif
if (dir)
pci_root_ops = dir;
else if (bios)
pci_root_ops = bios;
else {
printk("PCI: No PCI bus detected\n");
return;
}
printk("PCI: Probing PCI hardware\n");
pci_root_bus = pci_scan_bus(0, pci_root_ops, NULL);
pcibios_irq_init();
pcibios_fixup_peer_bridges();
pcibios_fixup_irqs();
pcibios_resource_survey();
}
char * __devinit pcibios_setup(char *str)
{
if (!strcmp(str, "off")) {
pci_probe = 0;
return NULL;
}
else if (!strncmp(str, "bios", 4)) {
printk("PCI: No PCI bios access on x86-64. BIOS hint ignored.\n");
return NULL;
} else if (!strcmp(str, "nobios")) {
pci_probe &= ~PCI_PROBE_BIOS;
return NULL;
} else if (!strcmp(str, "nosort")) { /* Default */
pci_probe |= PCI_NO_SORT;
return NULL;
}
#ifdef CONFIG_PCI_DIRECT
else if (!strcmp(str, "conf1")) {
pci_probe = PCI_PROBE_CONF1 | PCI_NO_CHECKS;
return NULL;
}
else if (!strcmp(str, "conf2")) {
pci_probe = PCI_PROBE_CONF2 | PCI_NO_CHECKS;
return NULL;
}
#endif
#ifdef CONFIG_ACPI_PCI
else if (!strcmp(str, "noacpi")) {
pci_probe |= PCI_NO_ACPI_ROUTING;
return NULL;
}
#endif
else if (!strcmp(str, "rom")) {
pci_probe |= PCI_ASSIGN_ROMS;
return NULL;
} else if (!strcmp(str, "assign-busses")) {
pci_probe |= PCI_ASSIGN_ALL_BUSSES;
return NULL;
} else if (!strncmp(str, "irqmask=", 8)) {
pcibios_irq_mask = simple_strtol(str+8, NULL, 0);
return NULL;
} else if (!strncmp(str, "lastbus=", 8)) {
pcibios_last_bus = simple_strtol(str+8, NULL, 0);
return NULL;
} else if (!strcmp(str, "usepirqmask")) {
pci_probe |= PCI_USE_PIRQ_MASK;
return NULL;
}
return str;
}
unsigned int pcibios_assign_all_busses(void)
{
return (pci_probe & PCI_ASSIGN_ALL_BUSSES) ? 1 : 0;
}
int pcibios_enable_device(struct pci_dev *dev)
{
int err;
if ((err = pcibios_enable_resources(dev)) < 0)
return err;
pcibios_enable_irq(dev);
return 0;
}
arch_initcall(pci_direct_init);
arch/x86_64/pci/fixup.c 0 → 100644
View file @ e4b9e2aa
/*
* Exceptions for specific devices. Usually work-arounds for fatal design flaws.
*
Short list on x86-64........so far.
*/
#include <linux/pci.h>
#include <linux/init.h>
#include "pci.h"
static void __devinit pci_fixup_ncr53c810(struct pci_dev *d)
{
/*
* NCR 53C810 returns class code 0 (at least on some systems).
* Fix class to be PCI_CLASS_STORAGE_SCSI
*/
if (!d->class) {
printk(KERN_WARNING "PCI: fixing NCR 53C810 class code for %s\n", d->slot_name);
d->class = PCI_CLASS_STORAGE_SCSI << 8;
}
}
static void __devinit pci_fixup_ide_bases(struct pci_dev *d)
{
int i;
/*
* PCI IDE controllers use non-standard I/O port decoding, respect it.
*/
if ((d->class >> 8) != PCI_CLASS_STORAGE_IDE)
return;
DBG("PCI: IDE base address fixup for %s\n", d->slot_name);
for(i=0; i<4; i++) {
struct resource *r = &d->resource[i];
if ((r->start & ~0x80) == 0x374) {
r->start |= 2;
r->end = r->start;
}
}
}
static void __devinit pci_fixup_ide_trash(struct pci_dev *d)
{
int i;
/*
* There exist PCI IDE controllers which have utter garbage
* in first four base registers. Ignore that.
*/
DBG("PCI: IDE base address trash cleared for %s\n", d->slot_name);
for(i=0; i<4; i++)
d->resource[i].start = d->resource[i].end = d->resource[i].flags = 0;
}
struct pci_fixup pcibios_fixups[] = {
{ PCI_FIXUP_HEADER, PCI_ANY_ID, PCI_ANY_ID, pci_fixup_ide_bases },
{ PCI_FIXUP_HEADER, PCI_VENDOR_ID_NCR, PCI_DEVICE_ID_NCR_53C810, pci_fixup_ncr53c810 },
{ 0 }
};
arch/x86_64/kernel/pci-irq.c → arch/x86_64/pci/irq.c
View file @ e4b9e2aa
......@@ -16,12 +16,12 @@
#include <asm/smp.h>
#include <asm/io_apic.h>
#include "pci-x86_64.h"
#include "pci.h"
#define PIRQ_SIGNATURE (('$' << 0) + ('P' << 8) + ('I' << 16) + ('R' << 24))
#define PIRQ_VERSION 0x0100
int pci_use_acpi_routing = 0;
int broken_hp_bios_irq9;
static struct irq_routing_table *pirq_table;
......@@ -44,6 +44,8 @@ struct irq_router {
int (*set)(struct pci_dev *router, struct pci_dev *dev, int pirq, int new);
};
int (*pcibios_enable_irq)(struct pci_dev *dev) = NULL;
/*
* Search 0xf0000 -- 0xfffff for the PCI IRQ Routing Table.
*/
......@@ -77,9 +79,6 @@ static struct irq_routing_table * __init pirq_find_routing_table(void)
* If we have a IRQ routing table, use it to search for peer host
* bridges. It's a gross hack, but since there are no other known
* ways how to get a list of buses, we have to go this way.
*
* [maybe x86-64 architecture should define way to query this info in
more reasonable way?]
*/
static void __init pirq_peer_trick(void)
......@@ -117,7 +116,7 @@ static void __init pirq_peer_trick(void)
* Code for querying and setting of IRQ routes on various interrupt routers.
*/
static void eisa_set_level_irq(unsigned int irq)
void eisa_set_level_irq(unsigned int irq)
{
unsigned char mask = 1 << (irq & 7);
unsigned int port = 0x4d0 + (irq >> 3);
......@@ -176,24 +175,6 @@ static int pirq_ali_set(struct pci_dev *router, struct pci_dev *dev, int pirq, i
return 0;
}
/*
* The Intel PIIX4 pirq rules are fairly simple: "pirq" is
* just a pointer to the config space.
*/
static int pirq_piix_get(struct pci_dev *router, struct pci_dev *dev, int pirq)
{
u8 x;
pci_read_config_byte(router, pirq, &x);
return (x < 16) ? x : 0;
}
static int pirq_piix_set(struct pci_dev *router, struct pci_dev *dev, int pirq, int irq)
{
pci_write_config_byte(router, pirq, irq);
return 1;
}
/*
* The VIA pirq rules are nibble-based, like ALI,
* but without the ugly irq number munging.
......@@ -209,35 +190,6 @@ static int pirq_via_set(struct pci_dev *router, struct pci_dev *dev, int pirq, i
return 1;
}
/*
* OPTI: high four bits are nibble pointer..
* I wonder what the low bits do?
*/
static int pirq_opti_get(struct pci_dev *router, struct pci_dev *dev, int pirq)
{
return read_config_nybble(router, 0xb8, pirq >> 4);
}
static int pirq_opti_set(struct pci_dev *router, struct pci_dev *dev, int pirq, int irq)
{
write_config_nybble(router, 0xb8, pirq >> 4, irq);
return 1;
}
/*
* Cyrix: nibble offset 0x5C
*/
static int pirq_cyrix_get(struct pci_dev *router, struct pci_dev *dev, int pirq)
{
return read_config_nybble(router, 0x5C, pirq-1);
}
static int pirq_cyrix_set(struct pci_dev *router, struct pci_dev *dev, int pirq, int irq)
{
write_config_nybble(router, 0x5C, pirq-1, irq);
return 1;
}
/*
* PIRQ routing for SiS 85C503 router used in several SiS chipsets
* According to the SiS 5595 datasheet (preliminary V1.0, 12/24/1997)
......@@ -290,7 +242,7 @@ static int pirq_sis_get(struct pci_dev *router, struct pci_dev *dev, int pirq)
case 0x43:
case 0x44:
case 0x62:
pci_read_config_byte(router, reg, &x);
pci_read_config_byte(router, reg, &x);
if (reg != 0x62)
break;
if (!(x & 0x40))
......@@ -344,57 +296,7 @@ static int pirq_sis_set(struct pci_dev *router, struct pci_dev *dev, int pirq, i
return 1;
}
/*
* VLSI: nibble offset 0x74 - educated guess due to routing table and
* config space of VLSI 82C534 PCI-bridge/router (1004:0102)
* Tested on HP OmniBook 800 covering PIRQ 1, 2, 4, 8 for onboard
* devices, PIRQ 3 for non-pci(!) soundchip and (untested) PIRQ 6
* for the busbridge to the docking station.
*/
static int pirq_vlsi_get(struct pci_dev *router, struct pci_dev *dev, int pirq)
{
if (pirq > 8) {
printk(KERN_INFO "VLSI router pirq escape (%d)\n", pirq);
return 0;
}
return read_config_nybble(router, 0x74, pirq-1);
}
static int pirq_vlsi_set(struct pci_dev *router, struct pci_dev *dev, int pirq, int irq)
{
if (pirq > 8) {
printk(KERN_INFO "VLSI router pirq escape (%d)\n", pirq);
return 0;
}
write_config_nybble(router, 0x74, pirq-1, irq);
return 1;
}
/*
* ServerWorks: PCI interrupts mapped to system IRQ lines through Index
* and Redirect I/O registers (0x0c00 and 0x0c01). The Index register
* format is (PCIIRQ## | 0x10), e.g.: PCIIRQ10=0x1a. The Redirect
* register is a straight binary coding of desired PIC IRQ (low nibble).
*
* The 'link' value in the PIRQ table is already in the correct format
* for the Index register. There are some special index values:
* 0x00 for ACPI (SCI), 0x01 for USB, 0x02 for IDE0, 0x04 for IDE1,
* and 0x03 for SMBus.
*/
static int pirq_serverworks_get(struct pci_dev *router, struct pci_dev *dev, int pirq)
{
outb_p(pirq, 0xc00);
return inb(0xc01) & 0xf;
}
static int pirq_serverworks_set(struct pci_dev *router, struct pci_dev *dev, int pirq, int irq)
{
outb_p(pirq, 0xc00);
outb_p(irq, 0xc01);
return 1;
}
#if 0 /* kept as reference */
/* Support for AMD756 PCI IRQ Routing
* Jhon H. Caicedo <jhcaiced@osso.org.co>
* Jun/21/2001 0.2.0 Release, fixed to use "nybble" functions... (jhcaiced)
......@@ -402,6 +304,8 @@ static int pirq_serverworks_set(struct pci_dev *router, struct pci_dev *dev, int
* The AMD756 pirq rules are nibble-based
* offset 0x56 0-3 PIRQA 4-7 PIRQB
* offset 0x57 0-3 PIRQC 4-7 PIRQD
*
* AMD8111 is similar NIY.
*/
static int pirq_amd756_get(struct pci_dev *router, struct pci_dev *dev, int pirq)
{
......@@ -426,47 +330,13 @@ static int pirq_amd756_set(struct pci_dev *router, struct pci_dev *dev, int pirq
}
return 1;
}
#ifdef CONFIG_PCI_BIOS
static int pirq_bios_set(struct pci_dev *router, struct pci_dev *dev, int pirq, int irq)
{
struct pci_dev *bridge;
int pin = pci_get_interrupt_pin(dev, &bridge);
return pcibios_set_irq_routing(bridge, pin, irq);
}
static struct irq_router pirq_bios_router =
{ "BIOS", 0, 0, NULL, pirq_bios_set };
#endif
static struct irq_router pirq_routers[] = {
{ "PIIX", PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82371FB_0, pirq_piix_get, pirq_piix_set },
{ "PIIX", PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82371SB_0, pirq_piix_get, pirq_piix_set },
{ "PIIX", PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82371AB_0, pirq_piix_get, pirq_piix_set },
{ "PIIX", PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82371MX, pirq_piix_get, pirq_piix_set },
{ "PIIX", PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82443MX_0, pirq_piix_get, pirq_piix_set },
{ "PIIX", PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82801BA_0, pirq_piix_get, pirq_piix_set },
{ "ALI", PCI_VENDOR_ID_AL, PCI_DEVICE_ID_AL_M1533, pirq_ali_get, pirq_ali_set },
{ "VIA", PCI_VENDOR_ID_VIA, PCI_DEVICE_ID_VIA_82C586_0, pirq_via_get, pirq_via_set },
{ "VIA", PCI_VENDOR_ID_VIA, PCI_DEVICE_ID_VIA_82C596, pirq_via_get, pirq_via_set },
{ "VIA", PCI_VENDOR_ID_VIA, PCI_DEVICE_ID_VIA_82C686, pirq_via_get, pirq_via_set },
{ "OPTI", PCI_VENDOR_ID_OPTI, PCI_DEVICE_ID_OPTI_82C700, pirq_opti_get, pirq_opti_set },
{ "NatSemi", PCI_VENDOR_ID_CYRIX, PCI_DEVICE_ID_CYRIX_5520, pirq_cyrix_get, pirq_cyrix_set },
{ "SIS", PCI_VENDOR_ID_SI, PCI_DEVICE_ID_SI_503, pirq_sis_get, pirq_sis_set },
{ "VLSI 82C534", PCI_VENDOR_ID_VLSI, PCI_DEVICE_ID_VLSI_82C534, pirq_vlsi_get, pirq_vlsi_set },
{ "ServerWorks", PCI_VENDOR_ID_SERVERWORKS, PCI_DEVICE_ID_SERVERWORKS_OSB4,
pirq_serverworks_get, pirq_serverworks_set },
{ "ServerWorks", PCI_VENDOR_ID_SERVERWORKS, PCI_DEVICE_ID_SERVERWORKS_CSB5,
pirq_serverworks_get, pirq_serverworks_set },
{ "AMD756 VIPER", PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_VIPER_740B,
pirq_amd756_get, pirq_amd756_set },
{ "default", 0, 0, NULL, NULL }
};
......@@ -506,7 +376,7 @@ static void __init pirq_find_router(void)
}
/* Match against router device entry? Use it as a fallback */
if (r->vendor == pirq_router_dev->vendor && r->device == pirq_router_dev->device) {
pirq_router = r;
pirq_router = r;
}
}
printk(KERN_INFO "PCI: Using IRQ router %s [%04x/%04x] at %s\n",
......@@ -532,54 +402,6 @@ static void pcibios_test_irq_handler(int irq, void *dev_id, struct pt_regs *regs
{
}
#ifdef CONFIG_ACPI_PCI
static int acpi_lookup_irq (
struct pci_dev *dev,
u8 pin,
int assign)
{
int result = 0;
int irq = 0;
/* TBD: Select IRQ from possible to improve routing performance. */
result = acpi_prt_get_irq(dev, pin, &irq);
if (!irq)
result = -ENODEV;
if (0 != result) {
printk(KERN_WARNING "PCI: No IRQ known for interrupt pin %c of device %s\n",
'A'+pin, dev->slot_name);
return result;
}
dev->irq = irq;
if (!assign) {
/* only check for the IRQ */
printk(KERN_INFO "PCI: Found IRQ %d for device %s\n", irq,
dev->slot_name);
return 1;
}
/* also assign an IRQ */
if (irq && (dev->class >> 8) != PCI_CLASS_DISPLAY_VGA) {
result = acpi_prt_set_irq(dev, pin, irq);
if (0 != result) {
printk(KERN_WARNING "PCI: Could not assign IRQ %d to device %s\n", irq, dev->slot_name);
return result;
}
eisa_set_level_irq(irq);
printk(KERN_INFO "PCI: Assigned IRQ %d for device %s\n", irq, dev->slot_name);
}
return 1;
}
#endif /* CONFIG_ACPI_PCI */
static int pcibios_lookup_irq(struct pci_dev *dev, int assign)
{
u8 pin;
......@@ -598,12 +420,6 @@ static int pcibios_lookup_irq(struct pci_dev *dev, int assign)
return 0;
}
pin = pin - 1;
#ifdef CONFIG_ACPI_PCI
/* Use ACPI to lookup IRQ */
if (pci_use_acpi_routing)
return acpi_lookup_irq(dev, pin, assign);
#endif
/* Find IRQ routing entry */
......@@ -625,6 +441,15 @@ static int pcibios_lookup_irq(struct pci_dev *dev, int assign)
DBG(" -> PIRQ %02x, mask %04x, excl %04x", pirq, mask, pirq_table->exclusive_irqs);
mask &= pcibios_irq_mask;
/* Work around broken HP Pavilion Notebooks which assign USB to
IRQ 9 even though it is actually wired to IRQ 11 */
if (broken_hp_bios_irq9 && pirq == 0x59 && dev->irq == 9) {
dev->irq = 11;
pci_write_config_byte(dev, PCI_INTERRUPT_LINE, 11);
r->set(pirq_router_dev, dev, pirq, 11);
}
/*
* Find the best IRQ to assign: use the one
* reported by the device if possible.
......@@ -703,28 +528,15 @@ static int pcibios_lookup_irq(struct pci_dev *dev, int assign)
return 1;
}
void __init pcibios_irq_init(void)
static int __init pcibios_irq_init(void)
{
DBG("PCI: IRQ init\n");
#ifdef CONFIG_ACPI_PCI
if (!(pci_probe & PCI_NO_ACPI_ROUTING)) {
if (acpi_prts.count) {
printk(KERN_INFO "PCI: Using ACPI for IRQ routing\n");
pci_use_acpi_routing = 1;
return;
}
else
printk(KERN_WARNING "PCI: Invalid ACPI-PCI IRQ routing table\n");
}
#endif
if (pcibios_enable_irq)
return 0;
pirq_table = pirq_find_routing_table();
#ifdef CONFIG_PCI_BIOS
if (!pirq_table && (pci_probe & PCI_BIOS_IRQ_SCAN))
pirq_table = pcibios_get_irq_routing_table();
#endif
if (pirq_table) {
pirq_peer_trick();
pirq_find_router();
......@@ -738,8 +550,15 @@ void __init pcibios_irq_init(void)
if (io_apic_assign_pci_irqs)
pirq_table = NULL;
}
pcibios_enable_irq = pirq_enable_irq;
pcibios_fixup_irqs();
return 0;
}
subsys_initcall(pcibios_irq_init);
void __init pcibios_fixup_irqs(void)
{
struct pci_dev *dev;
......@@ -815,19 +634,19 @@ void pcibios_penalize_isa_irq(int irq)
pirq_penalty[irq] += 100;
}
void pcibios_enable_irq(struct pci_dev *dev)
int pirq_enable_irq(struct pci_dev *dev)
{
u8 pin;
pci_read_config_byte(dev, PCI_INTERRUPT_PIN, &pin);
u8 pin;
pci_read_config_byte(dev, PCI_INTERRUPT_PIN, &pin);
if (pin && !pcibios_lookup_irq(dev, 1) && !dev->irq) {
char *msg;
if (io_apic_assign_pci_irqs)
msg = " Probably buggy MP table.";
else if (pci_probe & PCI_BIOS_IRQ_SCAN)
msg = "";
else
msg = " Please try using pci=biosirq.";
printk(KERN_WARNING "PCI: No IRQ known for interrupt pin %c of device %s.%s\n",
'A' + pin - 1, dev->slot_name, msg);
char *msg;
if (io_apic_assign_pci_irqs)
msg = " Probably buggy MP table.";
else
msg = "";
printk(KERN_WARNING "PCI: No IRQ known for interrupt pin %c of device %s.%s\n",
'A' + pin - 1, dev->slot_name, msg);
}
return 0;
}
arch/x86_64/pci/legacy.c 0 → 100644
View file @ e4b9e2aa
/*
* legacy.c - traditional, old school PCI bus probing
*/
#include <linux/pci.h>
#include <linux/init.h>
#include "pci.h"
/*
* Discover remaining PCI buses in case there are peer host bridges.
* We use the number of last PCI bus provided by the PCI BIOS.
*/
static void __devinit pcibios_fixup_peer_bridges(void)
{
int n;
struct pci_bus bus;
struct pci_dev dev;
u16 l;
if (pcibios_last_bus <= 0 || pcibios_last_bus >= 0xff)
return;
DBG("PCI: Peer bridge fixup\n");
for (n=0; n <= pcibios_last_bus; n++) {
if (pci_bus_exists(&pci_root_buses, n))
continue;
bus.number = n;
bus.ops = pci_root_ops;
dev.bus = &bus;
for(dev.devfn=0; dev.devfn<256; dev.devfn += 8)
if (!pci_read_config_word(&dev, PCI_VENDOR_ID, &l) &&
l != 0x0000 && l != 0xffff) {
DBG("Found device at %02x:%02x [%04x]\n", n, dev.devfn, l);
printk(KERN_INFO "PCI: Discovered peer bus %02x\n", n);
pci_scan_bus(n, pci_root_ops, NULL);
break;
}
}
}
static int __init pci_legacy_init(void)
{
if (!pci_root_ops) {
printk("PCI: System does not support PCI\n");
return 0;
}
if (pcibios_scanned++)
return 0;
printk("PCI: Probing PCI hardware\n");
pci_root_bus = pcibios_scan_root(0);
pcibios_fixup_peer_bridges();
return 0;
}
subsys_initcall(pci_legacy_init);
arch/x86_64/kernel/pci-x86_64.h → arch/x86_64/pci/pci.h
View file @ e4b9e2aa
/*
* Low-Level PCI Access for i386 machines.
* Low-Level PCI Access for x86-64 machines.
*
* (c) 1999 Martin Mares <mj@ucw.cz>
*/
......@@ -26,8 +26,6 @@
extern unsigned int pci_probe;
/* pci-i386.c */
extern unsigned int pcibios_max_latency;
void pcibios_resource_survey(void);
......@@ -39,9 +37,6 @@ extern int pcibios_last_bus;
extern struct pci_bus *pci_root_bus;
extern struct pci_ops *pci_root_ops;
struct irq_routing_table *pcibios_get_irq_routing_table(void);
int pcibios_set_irq_routing(struct pci_dev *dev, int pin, int irq);
/* pci-irq.c */
struct irq_info {
......@@ -69,8 +64,10 @@ struct irq_routing_table {
extern unsigned int pcibios_irq_mask;
extern int pci_use_acpi_routing;
extern int pcibios_scanned;
extern spinlock_t pci_config_lock;
void pcibios_irq_init(void);
void pcibios_fixup_irqs(void);
void pcibios_enable_irq(struct pci_dev *dev);
int pirq_enable_irq(struct pci_dev *dev);
extern int (*pcibios_enable_irq)(struct pci_dev *dev);
arch/x86_64/kernel/pci-x86_64.c → arch/x86_64/pci/x86-64.c
View file @ e4b9e2aa
......@@ -22,67 +22,6 @@
* PCI to PCI Bridge Specification
* PCI System Design Guide
*
*
* CHANGELOG :
* Jun 17, 1994 : Modified to accommodate the broken pre-PCI BIOS SPECIFICATION
* Revision 2.0 present on <thys@dennis.ee.up.ac.za>'s ASUS mainboard.
*
* Jan 5, 1995 : Modified to probe PCI hardware at boot time by Frederic
* Potter, potter@cao-vlsi.ibp.fr
*
* Jan 10, 1995 : Modified to store the information about configured pci
* devices into a list, which can be accessed via /proc/pci by
* Curtis Varner, cvarner@cs.ucr.edu
*
* Jan 12, 1995 : CPU-PCI bridge optimization support by Frederic Potter.
* Alpha version. Intel & UMC chipset support only.
*
* Apr 16, 1995 : Source merge with the DEC Alpha PCI support. Most of the code
* moved to drivers/pci/pci.c.
*
* Dec 7, 1996 : Added support for direct configuration access of boards
* with Intel compatible access schemes (tsbogend@alpha.franken.de)
*
* Feb 3, 1997 : Set internal functions to static, save/restore flags
* avoid dead locks reading broken PCI BIOS, werner@suse.de
*
* Apr 26, 1997 : Fixed case when there is BIOS32, but not PCI BIOS
* (mj@atrey.karlin.mff.cuni.cz)
*
* May 7, 1997 : Added some missing cli()'s. [mj]
*
* Jun 20, 1997 : Corrected problems in "conf1" type accesses.
* (paubert@iram.es)
*
* Aug 2, 1997 : Split to PCI BIOS handling and direct PCI access parts
* and cleaned it up... Martin Mares <mj@atrey.karlin.mff.cuni.cz>
*
* Feb 6, 1998 : No longer using BIOS to find devices and device classes. [mj]
*
* May 1, 1998 : Support for peer host bridges. [mj]
*
* Jun 19, 1998 : Changed to use spinlocks, so that PCI configuration space
* can be accessed from interrupts even on SMP systems. [mj]
*
* August 1998 : Better support for peer host bridges and more paranoid
* checks for direct hardware access. Ugh, this file starts to look as
* a large gallery of common hardware bug workarounds (watch the comments)
* -- the PCI specs themselves are sane, but most implementors should be
* hit hard with \hammer scaled \magstep5. [mj]
*
* Jan 23, 1999 : More improvements to peer host bridge logic. i450NX fixup. [mj]
*
* Feb 8, 1999 : Added UM8886BF I/O address fixup. [mj]
*
* August 1999 : New resource management and configuration access stuff. [mj]
*
* Sep 19, 1999 : Use PCI IRQ routing tables for detection of peer host bridges.
* Based on ideas by Chris Frantz and David Hinds. [mj]
*
* Sep 28, 1999 : Handle unreported/unassigned IRQs. Thanks to Shuu Yamaguchi
* for a lot of patience during testing. [mj]
*
* Oct 8, 1999 : Split to pci-i386.c, pci-pc.c and pci-visws.c. [mj]
*/
#include <linux/types.h>
......@@ -92,7 +31,7 @@
#include <linux/ioport.h>
#include <linux/errno.h>
#include "pci-x86_64.h"
#include "pci.h"
void
pcibios_update_resource(struct pci_dev *dev, struct resource *root,
......
arch/x86_64/tools/offset.c
View file @ e4b9e2aa
......@@ -24,7 +24,6 @@ int main(void)
output("#ifndef ASM_OFFSET_H\n");
output("#define ASM_OFFSET_H 1\n");
// task struct entries needed by entry.S
#define ENTRY(entry) outconst("#define tsk_" #entry " %0", offsetof(struct task_struct, entry))
ENTRY(state);
ENTRY(flags);
......@@ -43,24 +42,8 @@ int main(void)
ENTRY(irqcount);
ENTRY(cpunumber);
ENTRY(irqstackptr);
ENTRY(__softirq_pending);
ENTRY(__local_irq_count);
ENTRY(__local_bh_count);
ENTRY(__ksoftirqd_task);
ENTRY(level4_pgt);
ENTRY(me);
#undef ENTRY
output("#ifdef __ASSEMBLY__");
#define CONST(t) outconst("#define " #t " %0", t)
CONST(TASK_SIZE);
CONST(SIGCHLD);
CONST(CLONE_VFORK);
CONST(CLONE_VM);
#undef CONST
output("#endif");
output("#endif\n");
return(0);
}
arch/x86_64/vmlinux.lds
View file @ e4b9e2aa
......@@ -3,8 +3,8 @@
*/
OUTPUT_FORMAT("elf64-x86-64", "elf64-x86-64", "elf64-x86-64")
OUTPUT_ARCH(i386:x86-64)
jiffies = jiffies_64;
ENTRY(_start)
jiffies_64 = jiffies;
SECTIONS
{
. = 0xffffffff80100000;
......
include/asm-x86_64/apic.h
View file @ e4b9e2aa
......@@ -41,15 +41,9 @@ static __inline__ void apic_wait_icr_idle(void)
while ( apic_read( APIC_ICR ) & APIC_ICR_BUSY );
}
#ifdef CONFIG_X86_GOOD_APIC
# define FORCE_READ_AROUND_WRITE 0
# define apic_read_around(x)
# define apic_write_around(x,y) apic_write((x),(y))
#else
# define FORCE_READ_AROUND_WRITE 1
# define apic_read_around(x) apic_read(x)
# define apic_write_around(x,y) apic_write_atomic((x),(y))
#endif
#define FORCE_READ_AROUND_WRITE 0
#define apic_read_around(x)
#define apic_write_around(x,y) apic_write((x),(y))
static inline void ack_APIC_irq(void)
{
......
include/asm-x86_64/bitops.h
View file @ e4b9e2aa
......@@ -475,6 +475,9 @@ static __inline__ int ffs(int x)
#define minix_find_first_zero_bit(addr,size) \
find_first_zero_bit((void*)addr,size)
/* find last set bit */
#define fls(x) generic_fls(x)
#endif /* __KERNEL__ */
#endif /* _X86_64_BITOPS_H */
include/asm-x86_64/bootsetup.h
View file @ e4b9e2aa
......@@ -18,6 +18,7 @@ extern char x86_boot_params[2048];
#define SYS_DESC_TABLE (*(struct sys_desc_table_struct*)(PARAM+0xa0))
#define MOUNT_ROOT_RDONLY (*(unsigned short *) (PARAM+0x1F2))
#define RAMDISK_FLAGS (*(unsigned short *) (PARAM+0x1F8))
#define SAVED_VIDEO_MODE (*(unsigned short *) (PARAM+0x1FA))
#define ORIG_ROOT_DEV (*(unsigned short *) (PARAM+0x1FC))
#define AUX_DEVICE_INFO (*(unsigned char *) (PARAM+0x1FF))
#define LOADER_TYPE (*(unsigned char *) (PARAM+0x210))
......
include/asm-x86_64/checksum.h
View file @ e4b9e2aa
#ifndef _X86_64_CHECKSUM_H
#define _X86_64_CHECKSUM_H
#include <linux/in6.h>
/*
* Checksums for x86-64
* Copyright 2002 by Andi Kleen, SuSE Labs
* with some code from asm-i386/checksum.h
*/
#include <linux/compiler.h>
#include <asm/uaccess.h>
#include <asm/byteorder.h>
/**
* csum_fold - Fold and invert a 32bit checksum.
* sum: 32bit unfolded sum
*
* Fold a 32bit running checksum to 16bit and invert it. This is usually
* the last step before putting a checksum into a packet.
* Make sure not to mix with 64bit checksums.
*/
static inline unsigned int csum_fold(unsigned int sum)
{
__asm__(
" addl %1,%0\n"
" adcl $0xffff,%0"
: "=r" (sum)
: "r" (sum << 16), "0" (sum & 0xffff0000)
);
return (~sum) >> 16;
}
/*
* This is a version of ip_compute_csum() optimized for IP headers,
......@@ -10,28 +37,33 @@
* By Jorge Cwik <jorge@laser.satlink.net>, adapted for linux by
* Arnt Gulbrandsen.
*/
static inline unsigned short ip_fast_csum(unsigned char * iph,
unsigned int ihl) {
/**
* ip_fast_csum - Compute the IPv4 header checksum efficiently.
* iph: ipv4 header
* ihl: length of header / 4
*/
static inline unsigned short ip_fast_csum(unsigned char *iph, unsigned int ihl)
{
unsigned int sum;
__asm__ __volatile__(
"\n movl (%1), %0"
"\n subl $4, %2"
"\n jbe 2f"
"\n addl 4(%1), %0"
"\n adcl 8(%1), %0"
"\n adcl 12(%1), %0"
"\n1: adcl 16(%1), %0"
"\n lea 4(%1), %1"
"\n decl %2"
"\n jne 1b"
"\n adcl $0, %0"
"\n movl %0, %2"
"\n shrl $16, %0"
"\n addw %w2, %w0"
"\n adcl $0, %0"
"\n notl %0"
"\n2:"
asm( " movl (%1), %0\n"
" subl $4, %2\n"
" jbe 2f\n"
" addl 4(%1), %0\n"
" adcl 8(%1), %0\n"
" adcl 12(%1), %0\n"
"1: adcl 16(%1), %0\n"
" lea 4(%1), %1\n"
" decl %2\n"
" jne 1b\n"
" adcl $0, %0\n"
" movl %0, %2\n"
" shrl $16, %0\n"
" addw %w2, %w0\n"
" adcl $0, %0\n"
" notl %0\n"
"2:"
/* Since the input registers which are loaded with iph and ipl
are modified, we must also specify them as outputs, or gcc
will assume they contain their original values. */
......@@ -40,120 +72,111 @@ static inline unsigned short ip_fast_csum(unsigned char * iph,
return(sum);
}
/*
* Fold a partial checksum. Note this works on a 32bit unfolded checksum. Make sure
* to not mix with 64bit checksums!
/**
* csum_tcpup_nofold - Compute an IPv4 pseudo header checksum.
* @saddr: source address
* @daddr: destination address
* @len: length of packet
* @proto: ip protocol of packet
* @sum: initial sum to be added in (32bit unfolded)
*
* Returns the pseudo header checksum the input data. Result is
* 32bit unfolded.
*/
static inline unsigned int csum_fold(unsigned int sum)
static inline unsigned long
csum_tcpudp_nofold(unsigned saddr, unsigned daddr, unsigned short len,
unsigned short proto, unsigned int sum)
{
__asm__(
"\n addl %1,%0"
"\n adcl $0xffff,%0"
asm(" addl %1, %0\n"
" adcl %2, %0\n"
" adcl %3, %0\n"
" adcl $0, %0\n"
: "=r" (sum)
: "r" (sum << 16), "0" (sum & 0xffff0000)
);
return (~sum) >> 16;
}
static inline unsigned long csum_tcpudp_nofold(unsigned saddr,
unsigned daddr,
unsigned short len,
unsigned short proto,
unsigned int sum)
{
__asm__(
"\n addl %1, %0"
"\n adcl %2, %0"
"\n adcl %3, %0"
"\n adcl $0, %0"
: "=r" (sum)
: "g" (daddr), "g"(saddr), "g"((ntohs(len)<<16)+proto*256), "0"(sum));
: "g" (daddr), "g" (saddr), "g" ((ntohs(len)<<16)+proto*256), "0" (sum));
return sum;
}
/*
* computes the checksum of the TCP/UDP pseudo-header
* returns a 16-bit checksum, already complemented
/**
* csum_tcpup_magic - Compute an IPv4 pseudo header checksum.
* @saddr: source address
* @daddr: destination address
* @len: length of packet
* @proto: ip protocol of packet
* @sum: initial sum to be added in (32bit unfolded)
*
* Returns the 16bit pseudo header checksum the input data already
* complemented and ready to be filled in.
*/
static inline unsigned short int csum_tcpudp_magic(unsigned long saddr,
unsigned long daddr,
unsigned short len,
unsigned short proto,
unsigned int sum)
static inline unsigned short int
csum_tcpudp_magic(unsigned long saddr, unsigned long daddr,
unsigned short len, unsigned short proto, unsigned int sum)
{
return csum_fold(csum_tcpudp_nofold(saddr,daddr,len,proto,sum));
}
/*
* computes the checksum of a memory block at buff, length len,
* and adds in "sum" (32-bit)
*
* returns a 32-bit number suitable for feeding into itself
* or csum_tcpudp_magic
/**
* csum_partial - Compute an internet checksum.
* @buff: buffer to be checksummed
* @len: length of buffer.
* @sum: initial sum to be added in (32bit unfolded)
*
* this function must be called with even lengths, except
* for the last fragment, which may be odd
* Returns the 32bit unfolded internet checksum of the buffer.
* Before filling it in it needs to be csum_fold()'ed.
* buff should be aligned to a 64bit boundary if possible.
*/
extern unsigned int csum_partial(const unsigned char *buff, int len, unsigned int sum);
#define _HAVE_ARCH_COPY_AND_CSUM_FROM_USER 1
#define HAVE_CSUM_COPY_USER 1
/* Do not call this directly. Use the wrappers below */
extern unsigned long csum_partial_copy_generic(const char *src, const char *dst,
unsigned len,
unsigned sum,
int *src_err_ptr, int *dst_err_ptr);
extern unsigned int csum_partial_copy_from_user(const char *src, char *dst,
int len, unsigned int isum, int *errp);
extern unsigned int csum_partial_copy_to_user(const char *src, char *dst,
int len, unsigned int isum, int *errp);
extern unsigned int csum_partial_copy_nocheck(const char *src, char *dst, int len,
unsigned int sum);
/* Old names. To be removed. */
#define csum_and_copy_to_user csum_partial_copy_to_user
#define csum_and_copy_from_user csum_partial_copy_from_user
/**
* ip_compute_csum - Compute an 16bit IP checksum.
* @buff: buffer address.
* @len: length of buffer.
*
* it's best to have buff aligned on a 32-bit boundary
* Returns the 16bit folded/inverted checksum of the passed buffer.
* Ready to fill in.
*/
extern unsigned int csum_partial(const unsigned char * buff, int len, unsigned int sum);
extern unsigned short ip_compute_csum(unsigned char * buff, int len);
/*
* the same as csum_partial, but copies from src while it
* checksums
/**
* csum_ipv6_magic - Compute checksum of an IPv6 pseudo header.
* @saddr: source address
* @daddr: destination address
* @len: length of packet
* @proto: protocol of packet
* @sum: initial sum (32bit unfolded) to be added in
*
* here even more important to align src and dst on a 32-bit (or even
* better 64-bit) boundary
*/
unsigned int csum_partial_copy(const char *src, char *dst, int len, unsigned int sum);
/*
* this is a new version of the above that records errors it finds in *errp,
* but continues and zeros the rest of the buffer.
* Computes an IPv6 pseudo header checksum. This sum is added the checksum
* into UDP/TCP packets and contains some link layer information.
* Returns the unfolded 32bit checksum.
*/
unsigned int csum_partial_copy_from_user(const char *src, char *dst, int len, unsigned int sum, int *errp);
unsigned int csum_partial_copy_nocheck(const char *src, char *dst, int len, unsigned int sum);
struct in6_addr;
/*
* this routine is used for miscellaneous IP-like checksums, mainly
* in icmp.c
*/
extern unsigned short ip_compute_csum(unsigned char * buff, int len);
#define _HAVE_ARCH_IPV6_CSUM
static __inline__ unsigned short int csum_ipv6_magic(struct in6_addr *saddr,
struct in6_addr *daddr,
__u32 len,
unsigned short proto,
unsigned int sum)
{
__asm__(
"\n addl 0(%1), %0"
"\n adcl 4(%1), %0"
"\n adcl 8(%1), %0"
"\n adcl 12(%1), %0"
"\n adcl 0(%2), %0"
"\n adcl 4(%2), %0"
"\n adcl 8(%2), %0"
"\n adcl 12(%2), %0"
"\n adcl %3, %0"
"\n adcl %4, %0"
"\n adcl $0, %0"
: "=&r" (sum)
: "r" (saddr), "r" (daddr),
"r"(htonl(len)), "r"(htonl(proto)), "0"(sum));
return csum_fold(sum);
}
#define _HAVE_ARCH_IPV6_CSUM 1
extern unsigned short
csum_ipv6_magic(struct in6_addr *saddr, struct in6_addr *daddr,
__u32 len, unsigned short proto, unsigned int sum);
#endif
include/asm-x86_64/cpufeature.h
View file @ e4b9e2aa
......@@ -7,9 +7,6 @@
#ifndef __ASM_X8664_CPUFEATURE_H
#define __ASM_X8664_CPUFEATURE_H
/* Sample usage: CPU_FEATURE_P(cpu.x86_capability, FPU) */
#define CPU_FEATURE_P(CAP, FEATURE) test_bit(CAP, X86_FEATURE_##FEATURE ##_BIT)
#define NCAPINTS 4 /* Currently we have 4 32-bit words worth of info */
/* Intel-defined CPU features, CPUID level 0x00000001, word 0 */
......@@ -63,11 +60,26 @@
#define X86_FEATURE_CYRIX_ARR (3*32+ 2) /* Cyrix ARRs (= MTRRs) */
#define X86_FEATURE_CENTAUR_MCR (3*32+ 3) /* Centaur MCRs (= MTRRs) */
#endif /* __ASM_X8664_CPUFEATURE_H */
#define cpu_has(c, bit) test_bit(bit, (c)->x86_capability)
#define boot_cpu_has(bit) test_bit(bit, boot_cpu_data.x86_capability)
/*
* Local Variables:
* mode:c
* comment-column:42
* End:
*/
#define cpu_has_fpu 1
#define cpu_has_vme 0
#define cpu_has_de 1
#define cpu_has_pse 1
#define cpu_has_tsc 1
#define cpu_has_pae ___BUG___
#define cpu_has_pge 1
#define cpu_has_apic 1
#define cpu_has_mtrr 1
#define cpu_has_mmx 1
#define cpu_has_fxsr 1
#define cpu_has_xmm 1
#define cpu_has_ht 0 /* you need to report the support from i386. sorry */
#define cpu_has_mp 1 /* XXX */
#define cpu_has_k6_mtrr 0
#define cpu_has_cyrix_arr 0
#define cpu_has_centaur_mcr 0
#endif /* __ASM_X8664_CPUFEATURE_H */
include/asm-x86_64/desc.h
View file @ e4b9e2aa
......@@ -141,17 +141,17 @@ static inline void set_ldt_desc(unsigned n, void *addr, int size)
/*
* load one particular LDT into the current CPU
*/
extern inline void load_LDT (struct mm_struct *mm)
extern inline void load_LDT (mm_context_t *pc)
{
int cpu = smp_processor_id();
void *segments = mm->context.segments;
int count = pc->size;
if (!segments) {
if (!count) {
clear_LDT(cpu);
return;
}
set_ldt_desc(cpu, segments, LDT_ENTRIES);
set_ldt_desc(cpu, pc->ldt, count);
__load_LDT(cpu);
}
......
include/asm-x86_64/i387.h
View file @ e4b9e2aa
......@@ -140,6 +140,7 @@ static inline void empty_fpu(struct task_struct *child)
{
if (!child->used_math) {
/* Simulate an empty FPU. */
memset(&child->thread.i387.fxsave,0,sizeof(struct i387_fxsave_struct));
child->thread.i387.fxsave.cwd = 0x037f;
child->thread.i387.fxsave.swd = 0;
child->thread.i387.fxsave.twd = 0;
......
include/asm-x86_64/io.h
View file @ e4b9e2aa
......@@ -193,9 +193,9 @@ extern void iounmap(void *addr);
#define __raw_writew writew
#define __raw_writel writel
void *memcpy_fromio(void*,void*,unsigned);
void *memcpy_toio(void*,void*,unsigned);
#define memset_io(a,b,c) memset(__io_virt(a),(b),(c))
#define memcpy_fromio(a,b,c) memcpy((a),__io_virt(b),(c))
#define memcpy_toio(a,b,c) memcpy(__io_virt(a),(b),(c))
/*
* ISA space is 'always mapped' on a typical x86 system, no need to
......
include/asm-x86_64/ioctls.h
View file @ e4b9e2aa
......@@ -67,6 +67,7 @@
#define TIOCGICOUNT 0x545D /* read serial port inline interrupt counts */
#define TIOCGHAYESESP 0x545E /* Get Hayes ESP configuration */
#define TIOCSHAYESESP 0x545F /* Set Hayes ESP configuration */
#define FIOQSIZE 0x5460
/* Used for packet mode */
#define TIOCPKT_DATA 0
......
include/asm-x86_64/mmu.h
View file @ e4b9e2aa
#ifndef __x86_64_MMU_H
#define __x86_64_MMU_H
#include <linux/spinlock.h>
/*
* The x86_64 doesn't have a mmu context, but
* we put the segment information here.
*
* cpu_vm_mask is used to optimize ldt flushing.
*/
typedef struct {
void *segments;
unsigned long cpuvalid;
void *ldt;
rwlock_t ldtlock;
int size;
struct semaphore sem;
} mm_context_t;
#endif
include/asm-x86_64/mmu_context.h
View file @ e4b9e2aa
......@@ -13,7 +13,7 @@
* possibly do the LDT unload here?
*/
#define destroy_context(mm) do { } while(0)
#define init_new_context(tsk,mm) 0
int init_new_context(struct task_struct *tsk, struct mm_struct *mm);
#ifdef CONFIG_SMP
......@@ -34,20 +34,17 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
if (likely(prev != next)) {
/* stop flush ipis for the previous mm */
clear_bit(cpu, &prev->cpu_vm_mask);
/*
* Re-load LDT if necessary
*/
if (unlikely(prev->context.segments != next->context.segments))
load_LDT(next);
#ifdef CONFIG_SMP
cpu_tlbstate[cpu].state = TLBSTATE_OK;
cpu_tlbstate[cpu].active_mm = next;
#endif
set_bit(cpu, &next->cpu_vm_mask);
set_bit(cpu, &next->context.cpuvalid);
/* Re-load page tables */
*read_pda(level4_pgt) = __pa(next->pgd) | _PAGE_TABLE;
__flush_tlb();
if (next->context.size + prev->context.size)
load_LDT(&next->context);
}
#ifdef CONFIG_SMP
else {
......@@ -59,9 +56,8 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
* tlb flush IPI delivery. We must flush our tlb.
*/
local_flush_tlb();
load_LDT(&next->context);
}
if (!test_and_set_bit(cpu, &next->context.cpuvalid))
load_LDT(next);
}
#endif
}
......
include/asm-x86_64/page.h
View file @ e4b9e2aa
......@@ -9,23 +9,18 @@
#define PAGE_SIZE (1UL << PAGE_SHIFT)
#endif
#define PAGE_MASK (~(PAGE_SIZE-1))
#define PHYSICAL_PAGE_MASK (~(PAGE_SIZE-1) & (__PHYSICAL_MASK << PAGE_SHIFT))
#define THREAD_SIZE (2*PAGE_SIZE)
#define CURRENT_MASK (~(THREAD_SIZE-1))
#ifdef __KERNEL__
#ifndef __ASSEMBLY__
#if 0
#include <asm/mmx.h>
#define clear_page(page) mmx_clear_page((void *)(page))
#define copy_page(to,from) mmx_copy_page(to,from)
#else
#define clear_page(page) memset((void *)(page), 0, PAGE_SIZE)
#define copy_page(to,from) memcpy((void *)(to), (void *)(from), PAGE_SIZE)
#endif
void clear_page(void *);
void copy_page(void *, void *);
#define clear_user_page(page, vaddr) clear_page(page)
#define copy_user_page(to, from, vaddr) copy_page(to, from)
#define clear_user_page(page, vaddr, pg) clear_page(page)
#define copy_user_page(to, from, vaddr, pg) copy_page(to, from)
/*
* These are used to make use of C type-checking..
......@@ -34,7 +29,7 @@ typedef struct { unsigned long pte; } pte_t;
typedef struct { unsigned long pmd; } pmd_t;
typedef struct { unsigned long pgd; } pgd_t;
typedef struct { unsigned long pml4; } pml4_t;
#define PTE_MASK PAGE_MASK
#define PTE_MASK PHYSICAL_PAGE_MASK
typedef struct { unsigned long pgprot; } pgprot_t;
......@@ -59,9 +54,12 @@ typedef struct { unsigned long pgprot; } pgprot_t;
#define __START_KERNEL 0xffffffff80100000
#define __START_KERNEL_map 0xffffffff80000000
#define __PAGE_OFFSET 0x0000010000000000
#define __PHYSICAL_MASK 0x000000ffffffffff
#ifndef __ASSEMBLY__
#include <linux/stringify.h>
/*
* Tell the user there is some problem. The exception handler decodes this frame.
*/
......@@ -70,7 +68,9 @@ struct bug_frame {
char *filename; /* should use 32bit offset instead, but the assembler doesn't like it */
unsigned short line;
} __attribute__((packed));
#define BUG() asm volatile("ud2 ; .quad %c1 ; .short %c0" :: "i"(__LINE__), "i" (__FILE__))
#define BUG() \
asm volatile("ud2 ; .quad %c1 ; .short %c0" :: \
"i"(__LINE__), "i" (__stringify(KBUILD_BASENAME)))
#define PAGE_BUG(page) BUG()
void out_of_line_bug(void);
......@@ -103,8 +103,11 @@ extern __inline__ int get_order(unsigned long size)
__pa(v); })
#define __va(x) ((void *)((unsigned long)(x)+PAGE_OFFSET))
#define virt_to_page(kaddr) (mem_map + (__pa(kaddr) >> PAGE_SHIFT))
#define VALID_PAGE(page) ((page - mem_map) < max_mapnr)
#define pfn_to_page(pfn) (mem_map + (pfn))
#define page_to_pfn(page) ((unsigned long)((page) - mem_map))
#define virt_to_page(kaddr) pfn_to_page(__pa(kaddr) >> PAGE_SHIFT)
#define pfn_valid(pfn) ((pfn) < max_mapnr)
#define virt_addr_valid(kaddr) pfn_valid(__pa(kaddr) >> PAGE_SHIFT)
#define VM_DATA_DEFAULT_FLAGS (VM_READ | VM_WRITE | VM_EXEC | \
......
include/asm-x86_64/pci.h
View file @ e4b9e2aa
......@@ -2,7 +2,6 @@
#define __x8664_PCI_H
#include <linux/config.h>
#include <asm/io.h>
#ifdef __KERNEL__
......@@ -20,17 +19,19 @@ extern unsigned long pci_mem_start;
#define PCIBIOS_MIN_IO 0x1000
#define PCIBIOS_MIN_MEM (pci_mem_start)
void pcibios_config_init(void);
struct pci_bus * pcibios_scan_root(int bus);
extern int (*pci_config_read)(int seg, int bus, int dev, int fn, int reg, int len, u32 *value);
extern int (*pci_config_write)(int seg, int bus, int dev, int fn, int reg, int len, u32 value);
void pcibios_set_master(struct pci_dev *dev);
void pcibios_penalize_isa_irq(int irq);
struct irq_routing_table *pcibios_get_irq_routing_table(void);
int pcibios_set_irq_routing(struct pci_dev *dev, int pin, int irq);
/* Dynamic DMA mapping stuff.
* x8664 has everything mapped statically.
*/
#include <linux/types.h>
#include <linux/slab.h>
#include <linux/mm.h>
#include <linux/init.h>
#include <asm/scatterlist.h>
#include <linux/string.h>
#include <asm/io.h>
......
include/asm-x86_64/pda.h
View file @ e4b9e2aa
......@@ -2,19 +2,17 @@
#define X86_64_PDA_H
#include <linux/stddef.h>
#ifndef ASM_OFFSET_H
#include <asm/offset.h>
#endif
#include <linux/types.h>
#include <linux/cache.h>
/* Per processor datastructure. %gs points to it while the kernel runs */
/* To use a new field with the *_pda macros it needs to be added to tools/offset.c */
struct x8664_pda {
struct x8664_pda *me;
struct task_struct *pcurrent; /* Current process */
unsigned long cpudata_offset;
struct x8664_pda *me; /* Pointer to itself */
unsigned long kernelstack; /* TOS for current process */
unsigned long oldrsp; /* user rsp for system call */
unsigned long irqrsp; /* Old rsp for interrupts. */
struct task_struct *pcurrent; /* Current process */
int irqcount; /* Irq nesting counter. Starts with -1 */
int cpunumber; /* Logical CPU number */
char *irqstackptr; /* top of irqstack */
......@@ -39,18 +37,19 @@ extern struct x8664_pda cpu_pda[];
*/
#define sizeof_field(type,field) (sizeof(((type *)0)->field))
#define typeof_field(type,field) typeof(((type *)0)->field)
#ifndef __STR
#define __STR(x) #x
#endif
#define __STR2(x) __STR(x)
extern void __bad_pda_field(void);
#define pda_offset(field) offsetof(struct x8664_pda, field)
#define pda_to_op(op,field,val) do { \
switch (sizeof_field(struct x8664_pda, field)) { \
case 2: asm volatile(op "w %0,%%gs:" __STR2(pda_ ## field) ::"r" (val):"memory"); break; \
case 4: asm volatile(op "l %0,%%gs:" __STR2(pda_ ## field) ::"r" (val):"memory"); break; \
case 8: asm volatile(op "q %0,%%gs:" __STR2(pda_ ## field) ::"r" (val):"memory"); break; \
case 2: \
asm volatile(op "w %0,%%gs:%c1"::"r" (val),"i"(pda_offset(field)):"memory"); break; \
case 4: \
asm volatile(op "l %0,%%gs:%c1"::"r" (val),"i"(pda_offset(field)):"memory"); break; \
case 8: \
asm volatile(op "q %0,%%gs:%c1"::"r" (val),"i"(pda_offset(field)):"memory"); break; \
default: __bad_pda_field(); \
} \
} while (0)
......@@ -59,9 +58,12 @@ extern void __bad_pda_field(void);
#define pda_from_op(op,field) ({ \
typedef typeof_field(struct x8664_pda, field) T__; T__ ret__; \
switch (sizeof_field(struct x8664_pda, field)) { \
case 2: asm volatile(op "w %%gs:" __STR2(pda_ ## field) ",%0":"=r" (ret__)::"memory"); break; \
case 4: asm volatile(op "l %%gs:" __STR2(pda_ ## field) ",%0":"=r" (ret__)::"memory"); break; \
case 8: asm volatile(op "q %%gs:" __STR2(pda_ ## field) ",%0":"=r" (ret__)::"memory"); break; \
case 2: \
asm volatile(op "w %%gs:%c1,%0":"=r" (ret__):"i"(pda_offset(field)):"memory"); break;\
case 4: \
asm volatile(op "l %%gs:%c1,%0":"=r" (ret__):"i"(pda_offset(field)):"memory"); break;\
case 8: \
asm volatile(op "q %%gs:%c1,%0":"=r" (ret__):"i"(pda_offset(field)):"memory"); break;\
default: __bad_pda_field(); \
} \
ret__; })
......
include/asm-x86_64/percpu.h
View file @ e4b9e2aa
#ifndef __ARCH_I386_PERCPU__
#define __ARCH_I386_PERCPU__
#ifndef __ARCH_X8664_PERCPU__
#define __ARCH_X8664_PERCPU__
#include <asm-generic/percpu.h>
#include <asm/pda.h>
#endif /* __ARCH_I386_PERCPU__ */
/* var is in discarded region: offset to particular copy we want */
#define this_cpu(var) (*RELOC_HIDE(&var, read_pda(cpudata_offset)))
#define per_cpu(var, cpu) (*RELOC_HIDE(&var, per_cpu_pda[cpu]))
void setup_per_cpu_areas(void);
#endif /* __ARCH_X8664_PERCPU__ */
include/asm-x86_64/pgalloc.h
View file @ e4b9e2aa
......@@ -75,5 +75,7 @@ extern inline void pte_free(struct page *pte)
__free_page(pte);
}
#define pte_free_tlb(tlb,pte) tlb_remove_page((tlb),(pte))
#define pmd_free_tlb(tlb,x) do { } while (0)
#endif /* _X86_64_PGALLOC_H */
include/asm-x86_64/pgtable.h
View file @ e4b9e2aa
......@@ -65,6 +65,7 @@ extern unsigned long empty_zero_page[1024];
#define pgd_ERROR(e) \
printk("%s:%d: bad pgd %p(%016lx).\n", __FILE__, __LINE__, &(e), pgd_val(e))
#define pml4_none(x) (!pml4_val(x))
#define pgd_none(x) (!pgd_val(x))
......@@ -98,13 +99,8 @@ static inline void set_pml4(pml4_t *dst, pml4_t val)
#define pgd_page(pgd) \
((unsigned long) __va(pgd_val(pgd) & PAGE_MASK))
/* Find an entry in the second-level page table.. */
#define pmd_offset(dir, address) ((pmd_t *) pgd_page(*(dir)) + \
__pmd_offset(address))
#define ptep_get_and_clear(xp) __pte(xchg(&(xp)->pte, 0))
#define pte_same(a, b) ((a).pte == (b).pte)
#define __mk_pte(page_nr,pgprot) __pte(((page_nr) << PAGE_SHIFT) | pgprot_val(pgprot))
#define PMD_SIZE (1UL << PMD_SHIFT)
#define PMD_MASK (~(PMD_SIZE-1))
......@@ -202,7 +198,7 @@ static inline void set_pml4(pml4_t *dst, pml4_t val)
static inline unsigned long pgd_bad(pgd_t pgd)
{
unsigned long val = pgd_val(pgd);
val &= ~PAGE_MASK;
val &= ~PTE_MASK;
val &= ~(_PAGE_USER | _PAGE_DIRTY);
return val & ~(_PAGE_PRESENT | _PAGE_RW | _PAGE_ACCESSED);
}
......@@ -211,14 +207,18 @@ static inline unsigned long pgd_bad(pgd_t pgd)
#define pte_present(x) (pte_val(x) & (_PAGE_PRESENT | _PAGE_PROTNONE))
#define pte_clear(xp) do { set_pte(xp, __pte(0)); } while (0)
#define pmd_none(x) (!pmd_val(x))
#define pmd_present(x) (pmd_val(x) & _PAGE_PRESENT)
#define pmd_clear(xp) do { set_pmd(xp, __pmd(0)); } while (0)
#define pmd_bad(x) ((pmd_val(x) & (~PAGE_MASK & ~_PAGE_USER)) != _KERNPG_TABLE )
#define pages_to_mb(x) ((x) >> (20-PAGE_SHIFT)) /* FIXME: is this
right? */
#define pte_page(x) (mem_map+((unsigned long)((pte_val(x) >> PAGE_SHIFT))))
#define pte_page(x) pfn_to_page(pte_pfn(x))
#define pte_pfn(x) ((pte_val(x) >> PAGE_SHIFT) & __PHYSICAL_MASK)
static inline pte_t pfn_pte(unsigned long page_nr, pgprot_t pgprot)
{
pte_t pte;
pte_val(pte) = (page_nr << PAGE_SHIFT) | pgprot_val(pgprot);
return pte;
}
/*
* The following only work if pte_present() is true.
......@@ -256,7 +256,7 @@ static inline void ptep_mkdirty(pte_t *ptep) { set_bit(_PAGE_BIT_DIRTY, ptep);
* Level 4 access.
* Never use these in the common code.
*/
#define pml4_page(pml4) ((unsigned long) __va(pml4_val(pml4) & PAGE_MASK))
#define pml4_page(pml4) ((unsigned long) __va(pml4_val(pml4) & PTE_MASK))
#define pml4_index(address) ((address >> PML4_SHIFT) & (PTRS_PER_PML4-1))
#define pml4_offset_k(address) (init_level4_pgt + pml4_index(address))
#define mk_kernel_pml4(address) ((pml4_t){ (address) | _KERNPG_TABLE })
......@@ -277,41 +277,42 @@ static inline pgd_t *pgd_offset_k(unsigned long address)
pml4_t pml4;
pml4 = init_level4_pgt[pml4_index(address)];
return __pgd_offset_k(__va(pml4_val(pml4) & PAGE_MASK), address);
return __pgd_offset_k(__va(pml4_val(pml4) & PTE_MASK), address);
}
#define __pgd_offset(address) pgd_index(address)
#define pgd_offset(mm, address) ((mm)->pgd+pgd_index(address))
/* PMD - Level 2 access */
#define pmd_page_kernel(pmd) ((unsigned long) __va(pmd_val(pmd) & PAGE_MASK))
#define pmd_page(pmd) (mem_map + (pmd_val(pmd) >> PAGE_SHIFT))
#define __pmd_offset(address) \
(((address) >> PMD_SHIFT) & (PTRS_PER_PMD-1))
#define pmd_page_kernel(pmd) ((unsigned long) __va(pmd_val(pmd) & PTE_MASK))
#define pmd_page(pmd) (mem_map + ((pmd_val(pmd) & PTE_MASK)>>PAGE_SHIFT))
#define __pmd_offset(address) (((address) >> PMD_SHIFT) & (PTRS_PER_PMD-1))
#define pmd_offset(dir, address) ((pmd_t *) pgd_page(*(dir)) + \
__pmd_offset(address))
#define pmd_none(x) (!pmd_val(x))
#define pmd_present(x) (pmd_val(x) & _PAGE_PRESENT)
#define pmd_clear(xp) do { set_pmd(xp, __pmd(0)); } while (0)
#define pmd_bad(x) ((pmd_val(x) & (~PTE_MASK & ~_PAGE_USER)) != _KERNPG_TABLE )
#define pfn_pmd(nr,prot) (__pmd(((nr) << PAGE_SHIFT) | pgprot_val(prot)))
/* PTE - Level 1 access. */
#define mk_pte(page,pgprot) \
({ \
pte_t __pte; \
\
set_pte(&__pte, __pte(((page)-mem_map) * \
(unsigned long long)PAGE_SIZE + pgprot_val(pgprot))); \
__pte; \
})
/* page, protection -> pte */
#define mk_pte(page, pgprot) pfn_pte(page_to_pfn(page), (pgprot))
/* This takes a physical page address that is used by the remapping functions */
/* physical address -> PTE */
static inline pte_t mk_pte_phys(unsigned long physpage, pgprot_t pgprot)
{
pte_t __pte;
set_pte(&__pte, __pte(physpage + pgprot_val(pgprot)));
return __pte;
pte_t pte;
pte_val(pte) = physpage | pgprot_val(pgprot);
return pte;
}
/* Change flags of a PTE */
extern inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
{
set_pte(&pte, __pte((pte_val(pte) & _PAGE_CHG_MASK) | pgprot_val(newprot)));
pte_val(pte) &= _PAGE_CHG_MASK;
pte_val(pte) |= pgprot_val(newprot);
return pte;
}
......
include/asm-x86_64/processor.h
View file @ e4b9e2aa
......@@ -84,17 +84,6 @@ extern struct cpuinfo_x86 cpu_data[];
#define current_cpu_data boot_cpu_data
#endif
#define cpu_has_pge 1
#define cpu_has_pse 1
#define cpu_has_pae 1
#define cpu_has_tsc 1
#define cpu_has_de 1
#define cpu_has_vme 1
#define cpu_has_fxsr 1
#define cpu_has_mmx 1
#define cpu_has_xmm 1
#define cpu_has_apic 1
extern char ignore_irq13;
extern void identify_cpu(struct cpuinfo_x86 *);
......@@ -371,8 +360,6 @@ extern void release_thread(struct task_struct *);
*/
extern long kernel_thread(int (*fn)(void *), void * arg, unsigned long flags);
/* Copy and release all segment info associated with a VM */
extern void copy_segments(struct task_struct *p, struct mm_struct * mm);
extern void release_segments(struct mm_struct * mm);
/*
......
include/asm-x86_64/ptrace.h
View file @ e4b9e2aa
......@@ -85,6 +85,7 @@ struct pt_regs {
#define user_mode(regs) (!!((regs)->cs & 3))
#define instruction_pointer(regs) ((regs)->rip)
extern void show_regs(struct pt_regs *);
void signal_fault(struct pt_regs *regs, void *frame, char *where);
enum {
EF_CF = 0x00000001,
......
include/asm-x86_64/rwsem.h
View file @ e4b9e2aa
......@@ -146,18 +146,18 @@ static inline void __up_read(struct rw_semaphore *sem)
__s32 tmp = -RWSEM_ACTIVE_READ_BIAS;
__asm__ __volatile__(
"# beginning __up_read\n\t"
LOCK_PREFIX " xaddl %%edx,(%%rdi)\n\t" /* subtracts 1, returns the old value */
LOCK_PREFIX " xaddl %[tmp],(%%rdi)\n\t" /* subtracts 1, returns the old value */
" js 2f\n\t" /* jump if the lock is being waited upon */
"1:\n\t"
LOCK_SECTION_START("")
"2:\n\t"
" decw %%dx\n\t" /* do nothing if still outstanding active readers */
" decw %w[tmp]\n\t" /* do nothing if still outstanding active readers */
" jnz 1b\n\t"
" call rwsem_wake_thunk\n\t"
" jmp 1b\n"
LOCK_SECTION_END
"# ending __up_read\n"
: "+m"(sem->count), "+d"(tmp)
: "+m"(sem->count), "+r" (tmp)
: "D"(sem)
: "memory", "cc");
}
......@@ -167,23 +167,24 @@ LOCK_PREFIX " xaddl %%edx,(%%rdi)\n\t" /* subtracts 1, returns the old val
*/
static inline void __up_write(struct rw_semaphore *sem)
{
unsigned tmp;
__asm__ __volatile__(
"# beginning __up_write\n\t"
" movl %2,%%edx\n\t"
LOCK_PREFIX " xaddl %%edx,(%%rdi)\n\t" /* tries to transition 0xffff0001 -> 0x00000000 */
" movl %2,%[tmp]\n\t"
LOCK_PREFIX " xaddl %[tmp],(%%rdi)\n\t" /* tries to transition 0xffff0001 -> 0x00000000 */
" jnz 2f\n\t" /* jump if the lock is being waited upon */
"1:\n\t"
LOCK_SECTION_START("")
"2:\n\t"
" decw %%dx\n\t" /* did the active count reduce to 0? */
" decw %w[tmp]\n\t" /* did the active count reduce to 0? */
" jnz 1b\n\t" /* jump back if not */
" call rwsem_wake_thunk\n\t"
" jmp 1b\n"
LOCK_SECTION_END
"# ending __up_write\n"
: "+m"(sem->count)
: "+m"(sem->count), [tmp] "r" (tmp)
: "D"(sem), "i"(-RWSEM_ACTIVE_WRITE_BIAS)
: "memory", "cc", "rdx");
: "memory", "cc");
}
/*
......
include/asm-x86_64/signal.h
View file @ e4b9e2aa
#ifndef _ASMx8664_SIGNAL_H
#define _ASMx8664_SIGNAL_H
#ifndef __ASSEMBLY__
#include <linux/types.h>
#include <linux/linkage.h>
......@@ -33,6 +34,7 @@ asmlinkage int do_signal(struct pt_regs *regs, sigset_t *oldset);
typedef unsigned long sigset_t;
#endif /* __KERNEL__ */
#endif
#define SIGHUP 1
#define SIGINT 2
......@@ -131,6 +133,7 @@ typedef unsigned long sigset_t;
#define SIG_UNBLOCK 1 /* for unblocking signals */
#define SIG_SETMASK 2 /* for setting the signal mask */
#ifndef __ASSEMBLY__
/* Type of a signal handler. */
typedef void (*__sighandler_t)(int);
......@@ -200,6 +203,7 @@ extern __inline__ int sigfindinword(unsigned long word)
return word;
}
#endif
#endif
#endif /* __KERNEL__ */
#endif
include/asm-x86_64/softirq.h
View file @ e4b9e2aa
......@@ -26,7 +26,8 @@
"2: call do_softirq_thunk;" \
"jmp 1b;" \
".previous" \
:: "i" (pda___softirq_pending), "i" (pda___local_bh_count) : \
:: "i" (pda_offset(__softirq_pending)), \
"i" (pda_offset(__local_bh_count)) : \
"memory"); \
} while (0)
#define local_bh_enable() do { _local_bh_enable(); preempt_enable(); } while(0)
......
include/asm-x86_64/string.h
View file @ e4b9e2aa
......@@ -2,36 +2,76 @@
#define _X86_64_STRING_H_
#ifdef __KERNEL__
#include <linux/config.h>
#define struct_cpy(x,y) (*(x)=*(y))
#define __HAVE_ARCH_MEMCMP
#define __HAVE_ARCH_STRLEN
/* Written 2002 by Andi Kleen */
#define memset __builtin_memset
#define memcpy __builtin_memcpy
#define memcmp __builtin_memcmp
/* Work around "undefined reference to strlen" linker errors. */
/* #define strlen __builtin_strlen */
#define __HAVE_ARCH_STRLEN
static inline size_t strlen(const char * s)
/* Only used for special circumstances. Stolen from i386/string.h */
static inline void * __inline_memcpy(void * to, const void * from, size_t n)
{
int d0;
register int __res;
unsigned long d0, d1, d2;
__asm__ __volatile__(
"repne\n\t"
"scasb\n\t"
"notl %0\n\t"
"decl %0"
:"=c" (__res), "=&D" (d0) :"1" (s),"a" (0), "0" (0xffffffff));
return __res;
"rep ; movsl\n\t"
"testb $2,%b4\n\t"
"je 1f\n\t"
"movsw\n"
"1:\ttestb $1,%b4\n\t"
"je 2f\n\t"
"movsb\n"
"2:"
: "=&c" (d0), "=&D" (d1), "=&S" (d2)
:"0" (n/4), "q" (n),"1" ((long) to),"2" ((long) from)
: "memory");
return (to);
}
/* Even with __builtin_ the compiler may decide to use the out of line
function. */
#define __HAVE_ARCH_MEMCPY 1
extern void *__memcpy(void *to, const void *from, size_t len);
#define memcpy(dst,src,len) \
({ size_t __len = (len); \
void *__ret; \
if (__builtin_constant_p(len) && __len >= 64) \
__ret = __memcpy((dst),(src),__len); \
else \
__ret = __builtin_memcpy((dst),(src),__len); \
__ret; })
#if 0
#define __HAVE_ARCH_MEMSET
extern void *__memset(void *mem, int val, size_t len);
#define memset(dst,val,len) \
({ size_t __len = (len); \
void *__ret; \
if (__builtin_constant_p(len) && __len >= 64) \
__ret = __memset((dst),(val),__len); \
else \
__ret = __builtin_memset((dst),(val),__len); \
__ret; })
#endif
#define __HAVE_ARCH_MEMMOVE
void * memmove(void * dest,const void *src,size_t count);
/* Use C out of line version for memcmp */
#define memcmp __builtin_memcmp
int memcmp(const void * cs,const void * ct,size_t count);
/* out of line string functions use always C versions */
#define strlen __builtin_strlen
size_t strlen(const char * s);
#define strcpy __builtin_strcpy
char * strcpy(char * dest,const char *src);
#define strcat __builtin_strcat
char * strcat(char * dest, const char * src);
extern char *strstr(const char *cs, const char *ct);
#define strcmp __builtin_strcmp
int strcmp(const char * cs,const char * ct);
#endif /* __KERNEL__ */
......
include/asm-x86_64/thread_info.h
View file @ e4b9e2aa
......@@ -51,6 +51,8 @@ struct thread_info {
/* how to get the thread information struct from C */
#define THREAD_SIZE (2*PAGE_SIZE)
static inline struct thread_info *current_thread_info(void)
{
struct thread_info *ti;
......@@ -66,7 +68,6 @@ static inline struct thread_info *stack_thread_info(void)
}
/* thread information allocation */
#define THREAD_SIZE (2*PAGE_SIZE)
#define alloc_thread_info() ((struct thread_info *) __get_free_pages(GFP_KERNEL,1))
#define free_thread_info(ti) free_pages((unsigned long) (ti), 1)
#define get_thread_info(ti) get_task_struct((ti)->task)
......@@ -75,6 +76,7 @@ static inline struct thread_info *stack_thread_info(void)
#else /* !__ASSEMBLY__ */
/* how to get the thread information struct from ASM */
/* only works on the process stack. otherwise get it via the PDA. */
#define GET_THREAD_INFO(reg) \
movq $-8192, reg; \
andq %rsp, reg
......
include/asm-x86_64/timex.h
View file @ e4b9e2aa
......@@ -48,4 +48,6 @@ static inline cycles_t get_cycles (void)
extern unsigned int cpu_khz;
#define ARCH_HAS_JIFFIES_64
#endif
include/asm-x86_64/tlb.h
View file @ e4b9e2aa
#ifndef TLB_H
#define TLB_H 1
#define tlb_start_vma(tlb, vma) do { } while (0)
#define tlb_end_vma(tlb, vma) do { } while (0)
#define tlb_remove_tlb_entry(tlb, pte, address) do { } while (0)
#define tlb_flush(tlb) flush_tlb_mm((tlb)->mm)
#include <asm-generic/tlb.h>
#endif
include/asm-x86_64/types.h
View file @ e4b9e2aa
......@@ -17,8 +17,8 @@ typedef unsigned short __u16;
typedef __signed__ int __s32;
typedef unsigned int __u32;
typedef __signed__ long __s64;
typedef unsigned long __u64;
typedef __signed__ long long __s64;
typedef unsigned long long __u64;
/*
* These aren't exported outside the kernel to avoid name space clashes
......@@ -34,8 +34,8 @@ typedef unsigned short u16;
typedef signed int s32;
typedef unsigned int u32;
typedef signed long s64;
typedef unsigned long u64;
typedef signed long long s64;
typedef unsigned long long u64;
#define BITS_PER_LONG 64
......
include/asm-x86_64/uaccess.h
View file @ e4b9e2aa
......@@ -230,138 +230,19 @@ do { \
/*
* Copy To/From Userspace
*
* This relies on an optimized common worker function.
*
* Could do special inline versions for small constant copies, but avoid this
* for now. It's not clear it is worth it.
*/
/* Generic arbitrary sized copy. */
extern unsigned long copy_user_generic(void *to, const void *from, unsigned len);
/* Could do 8byte accesses, instead of 4bytes. */
/* Generic arbitrary sized copy. */
#define __copy_user(to,from,size) \
do { \
long __d0, __d1; \
__asm__ __volatile__( \
"0: rep; movsl\n" \
" movq %3,%0\n" \
"1: rep; movsb\n" \
"2:\n" \
".section .fixup,\"ax\"\n" \
"3: lea 0(%3,%0,4),%0\n" \
" jmp 2b\n" \
".previous\n" \
".section __ex_table,\"a\"\n" \
" .align 8\n" \
" .quad 0b,3b\n" \
" .quad 1b,2b\n" \
".previous" \
: "=&c"(size), "=&D" (__d0), "=&S" (__d1) \
: "r"(size & 3), "0"(size / 4), "1"(to), "2"(from) \
: "memory"); \
} while (0)
#define __copy_user_zeroing(to,from,size) \
do { \
long __d0, __d1; \
__asm__ __volatile__( \
"0: rep; movsl\n" \
" movq %3,%0\n" \
"1: rep; movsb\n" \
"2:\n" \
".section .fixup,\"ax\"\n" \
"3: lea 0(%3,%0,4),%0\n" \
"4: pushq %0\n" \
" pushq %%rax\n" \
" xorq %%rax,%%rax\n" \
" rep; stosb\n" \
" popq %%rax\n" \
" popq %0\n" \
" jmp 2b\n" \
".previous\n" \
".section __ex_table,\"a\"\n" \
" .align 8\n" \
" .quad 0b,3b\n" \
" .quad 1b,4b\n" \
".previous" \
: "=&c"(size), "=&D" (__d0), "=&S" (__d1) \
: "r"(size & 3), "0"(size / 4), "1"(to), "2"(from) \
: "memory"); \
} while (0)
/* We let the __ versions of copy_from/to_user inline, because they're often
* used in fast paths and have only a small space overhead.
*/
static inline unsigned long
__generic_copy_from_user_nocheck(void *to, const void *from, unsigned long n)
{
__copy_user_zeroing(to,from,n);
return n;
}
static inline unsigned long
__generic_copy_to_user_nocheck(void *to, const void *from, unsigned long n)
{
prefetch(from);
__copy_user(to,from,n);
return n;
}
unsigned long __generic_copy_to_user(void *, const void *, unsigned long);
unsigned long __generic_copy_from_user(void *, const void *, unsigned long);
static inline unsigned long
__constant_copy_to_user(void *to, const void *from, unsigned long n)
{
if (access_ok(VERIFY_WRITE, to, n))
__copy_user(to,from,n);
return n;
}
static inline unsigned long
__constant_copy_from_user(void *to, const void *from, unsigned long n)
{
if (access_ok(VERIFY_READ, from, n))
__copy_user_zeroing(to,from,n);
else
memset(to, 0, n);
return n;
}
static inline unsigned long
__constant_copy_to_user_nocheck(void *to, const void *from, unsigned long n)
{
__copy_user(to,from,n);
return n;
}
static inline unsigned long
__constant_copy_from_user_nocheck(void *to, const void *from, unsigned long n)
{
__copy_user_zeroing(to,from,n);
return n;
}
#define copy_to_user(to,from,n) \
(__builtin_constant_p(n) ? \
__constant_copy_to_user((to),(from),(n)) : \
__generic_copy_to_user((to),(from),(n)))
#define copy_from_user(to,from,n) \
(__builtin_constant_p(n) ? \
__constant_copy_from_user((to),(from),(n)) : \
__generic_copy_from_user((to),(from),(n)))
#define __copy_to_user(to,from,n) \
(__builtin_constant_p(n) ? \
__constant_copy_to_user_nocheck((to),(from),(n)) : \
__generic_copy_to_user_nocheck((to),(from),(n)))
#define __copy_from_user(to,from,n) \
(__builtin_constant_p(n) ? \
__constant_copy_from_user_nocheck((to),(from),(n)) : \
__generic_copy_from_user_nocheck((to),(from),(n)))
extern unsigned long copy_to_user(void *to, const void *from, unsigned len);
extern unsigned long copy_from_user(void *to, const void *from, unsigned len);
#define __copy_to_user copy_user_generic
#define __copy_from_user copy_user_generic
long strncpy_from_user(char *dst, const char *src, long count);
long __strncpy_from_user(char *dst, const char *src, long count);
......
include/asm-x86_64/user.h
View file @ e4b9e2aa
......@@ -84,6 +84,7 @@ struct user{
/* ptrace does not yet supply these. Someday.... */
int u_fpvalid; /* True if math co-processor being used. */
/* for this mess. Not yet used. */
int pad0;
struct user_i387_struct i387; /* Math Co-processor registers. */
/* The rest of this junk is to help gdb figure out what goes where */
unsigned long int u_tsize; /* Text segment size (pages). */
......@@ -96,12 +97,15 @@ struct user{
esp register. */
long int signal; /* Signal that caused the core dump. */
int reserved; /* No longer used */
int pad1;
struct user_pt_regs * u_ar0; /* Used by gdb to help find the values for */
/* the registers. */
struct user_i387_struct* u_fpstate; /* Math Co-processor pointer. */
unsigned long magic; /* To uniquely identify a core file */
char u_comm[32]; /* User command that was responsible */
unsigned long u_debugreg[8];
unsigned long error_code; /* CPU error code or 0 */
unsigned long fault_address; /* CR3 or 0 */
};
#define NBPG PAGE_SIZE
#define UPAGES 1
......
include/asm-x86_64/xor.h
View file @ e4b9e2aa
/*
* include/asm-i386/xor.h
* include/asm-x86_64/xor.h
*
* Optimized RAID-5 checksumming functions for MMX and SSE.
*
......@@ -13,525 +13,32 @@
* Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
*/
/*
* High-speed RAID5 checksumming functions utilizing MMX instructions.
* Copyright (C) 1998 Ingo Molnar.
* Cache avoiding checksumming functions utilizing KNI instructions
* Copyright (C) 1999 Zach Brown (with obvious credit due Ingo)
*/
#define FPU_SAVE \
do { \
if (!test_thread_flag(TIF_USEDFPU)) \
__asm__ __volatile__ (" clts;\n"); \
__asm__ __volatile__ ("fsave %0; fwait": "=m"(fpu_save[0])); \
} while (0)
#define FPU_RESTORE \
do { \
__asm__ __volatile__ ("frstor %0": : "m"(fpu_save[0])); \
if (!test_thread_flag(TIF_USEDFPU)) \
stts(); \
} while (0)
#define LD(x,y) " movq 8*("#x")(%1), %%mm"#y" ;\n"
#define ST(x,y) " movq %%mm"#y", 8*("#x")(%1) ;\n"
#define XO1(x,y) " pxor 8*("#x")(%2), %%mm"#y" ;\n"
#define XO2(x,y) " pxor 8*("#x")(%3), %%mm"#y" ;\n"
#define XO3(x,y) " pxor 8*("#x")(%4), %%mm"#y" ;\n"
#define XO4(x,y) " pxor 8*("#x")(%5), %%mm"#y" ;\n"
static void
xor_pII_mmx_2(unsigned long bytes, unsigned long *p1, unsigned long *p2)
{
unsigned long lines = bytes >> 7;
char fpu_save[108];
FPU_SAVE;
__asm__ __volatile__ (
#undef BLOCK
#define BLOCK(i) \
LD(i,0) \
LD(i+1,1) \
LD(i+2,2) \
LD(i+3,3) \
XO1(i,0) \
ST(i,0) \
XO1(i+1,1) \
ST(i+1,1) \
XO1(i+2,2) \
ST(i+2,2) \
XO1(i+3,3) \
ST(i+3,3)
" .align 32 ;\n"
" 1: ;\n"
BLOCK(0)
BLOCK(4)
BLOCK(8)
BLOCK(12)
" addq $128, %1 ;\n"
" addq $128, %2 ;\n"
" decq %0 ;\n"
" jnz 1b ;\n"
:
: "r" (lines),
"r" (p1), "r" (p2)
: "memory");
FPU_RESTORE;
}
static void
xor_pII_mmx_3(unsigned long bytes, unsigned long *p1, unsigned long *p2,
unsigned long *p3)
{
unsigned long lines = bytes >> 7;
char fpu_save[108];
FPU_SAVE;
__asm__ __volatile__ (
#undef BLOCK
#define BLOCK(i) \
LD(i,0) \
LD(i+1,1) \
LD(i+2,2) \
LD(i+3,3) \
XO1(i,0) \
XO1(i+1,1) \
XO1(i+2,2) \
XO1(i+3,3) \
XO2(i,0) \
ST(i,0) \
XO2(i+1,1) \
ST(i+1,1) \
XO2(i+2,2) \
ST(i+2,2) \
XO2(i+3,3) \
ST(i+3,3)
" .align 32 ;\n"
" 1: ;\n"
BLOCK(0)
BLOCK(4)
BLOCK(8)
BLOCK(12)
" addq $128, %1 ;\n"
" addq $128, %2 ;\n"
" addq $128, %3 ;\n"
" decq %0 ;\n"
" jnz 1b ;\n"
:
: "r" (lines),
"r" (p1), "r" (p2), "r" (p3)
: "memory");
FPU_RESTORE;
}
static void
xor_pII_mmx_4(unsigned long bytes, unsigned long *p1, unsigned long *p2,
unsigned long *p3, unsigned long *p4)
{
unsigned long lines = bytes >> 7;
char fpu_save[108];
FPU_SAVE;
__asm__ __volatile__ (
#undef BLOCK
#define BLOCK(i) \
LD(i,0) \
LD(i+1,1) \
LD(i+2,2) \
LD(i+3,3) \
XO1(i,0) \
XO1(i+1,1) \
XO1(i+2,2) \
XO1(i+3,3) \
XO2(i,0) \
XO2(i+1,1) \
XO2(i+2,2) \
XO2(i+3,3) \
XO3(i,0) \
ST(i,0) \
XO3(i+1,1) \
ST(i+1,1) \
XO3(i+2,2) \
ST(i+2,2) \
XO3(i+3,3) \
ST(i+3,3)
" .align 32 ;\n"
" 1: ;\n"
BLOCK(0)
BLOCK(4)
BLOCK(8)
BLOCK(12)
" addq $128, %1 ;\n"
" addq $128, %2 ;\n"
" addq $128, %3 ;\n"
" addq $128, %4 ;\n"
" decq %0 ;\n"
" jnz 1b ;\n"
:
: "r" (lines),
"r" (p1), "r" (p2), "r" (p3), "r" (p4)
: "memory");
FPU_RESTORE;
}
static void
xor_pII_mmx_5(unsigned long bytes, unsigned long *p1, unsigned long *p2,
unsigned long *p3, unsigned long *p4, unsigned long *p5)
{
unsigned long lines = bytes >> 7;
char fpu_save[108];
FPU_SAVE;
__asm__ __volatile__ (
#undef BLOCK
#define BLOCK(i) \
LD(i,0) \
LD(i+1,1) \
LD(i+2,2) \
LD(i+3,3) \
XO1(i,0) \
XO1(i+1,1) \
XO1(i+2,2) \
XO1(i+3,3) \
XO2(i,0) \
XO2(i+1,1) \
XO2(i+2,2) \
XO2(i+3,3) \
XO3(i,0) \
XO3(i+1,1) \
XO3(i+2,2) \
XO3(i+3,3) \
XO4(i,0) \
ST(i,0) \
XO4(i+1,1) \
ST(i+1,1) \
XO4(i+2,2) \
ST(i+2,2) \
XO4(i+3,3) \
ST(i+3,3)
" .align 32 ;\n"
" 1: ;\n"
BLOCK(0)
BLOCK(4)
BLOCK(8)
BLOCK(12)
" addq $128, %1 ;\n"
" addq $128, %2 ;\n"
" addq $128, %3 ;\n"
" addq $128, %4 ;\n"
" addq $128, %5 ;\n"
" decq %0 ;\n"
" jnz 1b ;\n"
:
: "g" (lines),
"r" (p1), "r" (p2), "r" (p3), "r" (p4), "r" (p5)
: "memory");
FPU_RESTORE;
}
#undef LD
#undef XO1
#undef XO2
#undef XO3
#undef XO4
#undef ST
#undef BLOCK
static void
xor_p5_mmx_2(unsigned long bytes, unsigned long *p1, unsigned long *p2)
{
unsigned long lines = bytes >> 6;
char fpu_save[108];
FPU_SAVE;
__asm__ __volatile__ (
" .align 32 ;\n"
" 1: ;\n"
" movq (%1), %%mm0 ;\n"
" movq 8(%1), %%mm1 ;\n"
" pxor (%2), %%mm0 ;\n"
" movq 16(%1), %%mm2 ;\n"
" movq %%mm0, (%1) ;\n"
" pxor 8(%2), %%mm1 ;\n"
" movq 24(%1), %%mm3 ;\n"
" movq %%mm1, 8(%1) ;\n"
" pxor 16(%2), %%mm2 ;\n"
" movq 32(%1), %%mm4 ;\n"
" movq %%mm2, 16(%1) ;\n"
" pxor 24(%2), %%mm3 ;\n"
" movq 40(%1), %%mm5 ;\n"
" movq %%mm3, 24(%1) ;\n"
" pxor 32(%2), %%mm4 ;\n"
" movq 48(%1), %%mm6 ;\n"
" movq %%mm4, 32(%1) ;\n"
" pxor 40(%2), %%mm5 ;\n"
" movq 56(%1), %%mm7 ;\n"
" movq %%mm5, 40(%1) ;\n"
" pxor 48(%2), %%mm6 ;\n"
" pxor 56(%2), %%mm7 ;\n"
" movq %%mm6, 48(%1) ;\n"
" movq %%mm7, 56(%1) ;\n"
" addq $64, %1 ;\n"
" addq $64, %2 ;\n"
" decq %0 ;\n"
" jnz 1b ;\n"
:
: "r" (lines),
"r" (p1), "r" (p2)
: "memory");
FPU_RESTORE;
}
static void
xor_p5_mmx_3(unsigned long bytes, unsigned long *p1, unsigned long *p2,
unsigned long *p3)
{
unsigned long lines = bytes >> 6;
char fpu_save[108];
FPU_SAVE;
__asm__ __volatile__ (
" .align 32,0x90 ;\n"
" 1: ;\n"
" movq (%1), %%mm0 ;\n"
" movq 8(%1), %%mm1 ;\n"
" pxor (%2), %%mm0 ;\n"
" movq 16(%1), %%mm2 ;\n"
" pxor 8(%2), %%mm1 ;\n"
" pxor (%3), %%mm0 ;\n"
" pxor 16(%2), %%mm2 ;\n"
" movq %%mm0, (%1) ;\n"
" pxor 8(%3), %%mm1 ;\n"
" pxor 16(%3), %%mm2 ;\n"
" movq 24(%1), %%mm3 ;\n"
" movq %%mm1, 8(%1) ;\n"
" movq 32(%1), %%mm4 ;\n"
" movq 40(%1), %%mm5 ;\n"
" pxor 24(%2), %%mm3 ;\n"
" movq %%mm2, 16(%1) ;\n"
" pxor 32(%2), %%mm4 ;\n"
" pxor 24(%3), %%mm3 ;\n"
" pxor 40(%2), %%mm5 ;\n"
" movq %%mm3, 24(%1) ;\n"
" pxor 32(%3), %%mm4 ;\n"
" pxor 40(%3), %%mm5 ;\n"
" movq 48(%1), %%mm6 ;\n"
" movq %%mm4, 32(%1) ;\n"
" movq 56(%1), %%mm7 ;\n"
" pxor 48(%2), %%mm6 ;\n"
" movq %%mm5, 40(%1) ;\n"
" pxor 56(%2), %%mm7 ;\n"
" pxor 48(%3), %%mm6 ;\n"
" pxor 56(%3), %%mm7 ;\n"
" movq %%mm6, 48(%1) ;\n"
" movq %%mm7, 56(%1) ;\n"
" addq $64, %1 ;\n"
" addq $64, %2 ;\n"
" addq $64, %3 ;\n"
" decq %0 ;\n"
" jnz 1b ;\n"
:
: "r" (lines),
"r" (p1), "r" (p2), "r" (p3)
: "memory" );
FPU_RESTORE;
}
static void
xor_p5_mmx_4(unsigned long bytes, unsigned long *p1, unsigned long *p2,
unsigned long *p3, unsigned long *p4)
{
unsigned long lines = bytes >> 6;
char fpu_save[108];
FPU_SAVE;
__asm__ __volatile__ (
" .align 32,0x90 ;\n"
" 1: ;\n"
" movq (%1), %%mm0 ;\n"
" movq 8(%1), %%mm1 ;\n"
" pxor (%2), %%mm0 ;\n"
" movq 16(%1), %%mm2 ;\n"
" pxor 8(%2), %%mm1 ;\n"
" pxor (%3), %%mm0 ;\n"
" pxor 16(%2), %%mm2 ;\n"
" pxor 8(%3), %%mm1 ;\n"
" pxor (%4), %%mm0 ;\n"
" movq 24(%1), %%mm3 ;\n"
" pxor 16(%3), %%mm2 ;\n"
" pxor 8(%4), %%mm1 ;\n"
" movq %%mm0, (%1) ;\n"
" movq 32(%1), %%mm4 ;\n"
" pxor 24(%2), %%mm3 ;\n"
" pxor 16(%4), %%mm2 ;\n"
" movq %%mm1, 8(%1) ;\n"
" movq 40(%1), %%mm5 ;\n"
" pxor 32(%2), %%mm4 ;\n"
" pxor 24(%3), %%mm3 ;\n"
" movq %%mm2, 16(%1) ;\n"
" pxor 40(%2), %%mm5 ;\n"
" pxor 32(%3), %%mm4 ;\n"
" pxor 24(%4), %%mm3 ;\n"
" movq %%mm3, 24(%1) ;\n"
" movq 56(%1), %%mm7 ;\n"
" movq 48(%1), %%mm6 ;\n"
" pxor 40(%3), %%mm5 ;\n"
" pxor 32(%4), %%mm4 ;\n"
" pxor 48(%2), %%mm6 ;\n"
" movq %%mm4, 32(%1) ;\n"
" pxor 56(%2), %%mm7 ;\n"
" pxor 40(%4), %%mm5 ;\n"
" pxor 48(%3), %%mm6 ;\n"
" pxor 56(%3), %%mm7 ;\n"
" movq %%mm5, 40(%1) ;\n"
" pxor 48(%4), %%mm6 ;\n"
" pxor 56(%4), %%mm7 ;\n"
" movq %%mm6, 48(%1) ;\n"
" movq %%mm7, 56(%1) ;\n"
" addq $64, %1 ;\n"
" addq $64, %2 ;\n"
" addq $64, %3 ;\n"
" addq $64, %4 ;\n"
" decq %0 ;\n"
" jnz 1b ;\n"
:
: "r" (lines),
"r" (p1), "r" (p2), "r" (p3), "r" (p4)
: "memory");
FPU_RESTORE;
}
static void
xor_p5_mmx_5(unsigned long bytes, unsigned long *p1, unsigned long *p2,
unsigned long *p3, unsigned long *p4, unsigned long *p5)
{
unsigned long lines = bytes >> 6;
char fpu_save[108];
FPU_SAVE;
__asm__ __volatile__ (
" .align 32,0x90 ;\n"
" 1: ;\n"
" movq (%1), %%mm0 ;\n"
" movq 8(%1), %%mm1 ;\n"
" pxor (%2), %%mm0 ;\n"
" pxor 8(%2), %%mm1 ;\n"
" movq 16(%1), %%mm2 ;\n"
" pxor (%3), %%mm0 ;\n"
" pxor 8(%3), %%mm1 ;\n"
" pxor 16(%2), %%mm2 ;\n"
" pxor (%4), %%mm0 ;\n"
" pxor 8(%4), %%mm1 ;\n"
" pxor 16(%3), %%mm2 ;\n"
" movq 24(%1), %%mm3 ;\n"
" pxor (%5), %%mm0 ;\n"
" pxor 8(%5), %%mm1 ;\n"
" movq %%mm0, (%1) ;\n"
" pxor 16(%4), %%mm2 ;\n"
" pxor 24(%2), %%mm3 ;\n"
" movq %%mm1, 8(%1) ;\n"
" pxor 16(%5), %%mm2 ;\n"
" pxor 24(%3), %%mm3 ;\n"
" movq 32(%1), %%mm4 ;\n"
" movq %%mm2, 16(%1) ;\n"
" pxor 24(%4), %%mm3 ;\n"
" pxor 32(%2), %%mm4 ;\n"
" movq 40(%1), %%mm5 ;\n"
" pxor 24(%5), %%mm3 ;\n"
" pxor 32(%3), %%mm4 ;\n"
" pxor 40(%2), %%mm5 ;\n"
" movq %%mm3, 24(%1) ;\n"
" pxor 32(%4), %%mm4 ;\n"
" pxor 40(%3), %%mm5 ;\n"
" movq 48(%1), %%mm6 ;\n"
" movq 56(%1), %%mm7 ;\n"
" pxor 32(%5), %%mm4 ;\n"
" pxor 40(%4), %%mm5 ;\n"
" pxor 48(%2), %%mm6 ;\n"
" pxor 56(%2), %%mm7 ;\n"
" movq %%mm4, 32(%1) ;\n"
" pxor 48(%3), %%mm6 ;\n"
" pxor 56(%3), %%mm7 ;\n"
" pxor 40(%5), %%mm5 ;\n"
" pxor 48(%4), %%mm6 ;\n"
" pxor 56(%4), %%mm7 ;\n"
" movq %%mm5, 40(%1) ;\n"
" pxor 48(%5), %%mm6 ;\n"
" pxor 56(%5), %%mm7 ;\n"
" movq %%mm6, 48(%1) ;\n"
" movq %%mm7, 56(%1) ;\n"
" addq $64, %1 ;\n"
" addq $64, %2 ;\n"
" addq $64, %3 ;\n"
" addq $64, %4 ;\n"
" addq $64, %5 ;\n"
" decq %0 ;\n"
" jnz 1b ;\n"
:
: "g" (lines),
"r" (p1), "r" (p2), "r" (p3), "r" (p4), "r" (p5)
: "memory");
FPU_RESTORE;
}
static struct xor_block_template xor_block_pII_mmx = {
name: "pII_mmx",
do_2: xor_pII_mmx_2,
do_3: xor_pII_mmx_3,
do_4: xor_pII_mmx_4,
do_5: xor_pII_mmx_5,
};
static struct xor_block_template xor_block_p5_mmx = {
name: "p5_mmx",
do_2: xor_p5_mmx_2,
do_3: xor_p5_mmx_3,
do_4: xor_p5_mmx_4,
do_5: xor_p5_mmx_5,
};
#undef FPU_SAVE
#undef FPU_RESTORE
/*
* Based on
* High-speed RAID5 checksumming functions utilizing SSE instructions.
* Copyright (C) 1998 Ingo Molnar.
*/
/*
* Cache avoiding checksumming functions utilizing KNI instructions
* Copyright (C) 1999 Zach Brown (with obvious credit due Ingo)
* x86-64 changes / gcc fixes from Andi Kleen.
* Copyright 2002 Andi Kleen, SuSE Labs.
*
* This hasn't been optimized for the hammer yet, but there are likely
* no advantages to be gotten from x86-64 here anyways.
*/
typedef struct { unsigned long a,b; } __attribute__((aligned(16))) xmm_store_t;
/* Doesn't use gcc to save the XMM registers, because there is no easy way to
tell it to do a clts before the register saving. */
#define XMMS_SAVE \
__asm__ __volatile__ ( \
asm volatile ( \
"movq %%cr0,%0 ;\n\t" \
"clts ;\n\t" \
"movups %%xmm0,(%1) ;\n\t" \
......@@ -543,7 +50,7 @@ static struct xor_block_template xor_block_p5_mmx = {
: "memory")
#define XMMS_RESTORE \
__asm__ __volatile__ ( \
asm volatile ( \
"sfence ;\n\t" \
"movups (%1),%%xmm0 ;\n\t" \
"movups 0x10(%1),%%xmm1 ;\n\t" \
......@@ -556,31 +63,31 @@ static struct xor_block_template xor_block_p5_mmx = {
#define OFFS(x) "16*("#x")"
#define PF_OFFS(x) "256+16*("#x")"
#define PF0(x) " prefetchnta "PF_OFFS(x)"(%1) ;\n"
#define LD(x,y) " movaps "OFFS(x)"(%1), %%xmm"#y" ;\n"
#define ST(x,y) " movaps %%xmm"#y", "OFFS(x)"(%1) ;\n"
#define PF1(x) " prefetchnta "PF_OFFS(x)"(%2) ;\n"
#define PF2(x) " prefetchnta "PF_OFFS(x)"(%3) ;\n"
#define PF3(x) " prefetchnta "PF_OFFS(x)"(%4) ;\n"
#define PF4(x) " prefetchnta "PF_OFFS(x)"(%5) ;\n"
#define PF5(x) " prefetchnta "PF_OFFS(x)"(%6) ;\n"
#define XO1(x,y) " xorps "OFFS(x)"(%2), %%xmm"#y" ;\n"
#define XO2(x,y) " xorps "OFFS(x)"(%3), %%xmm"#y" ;\n"
#define XO3(x,y) " xorps "OFFS(x)"(%4), %%xmm"#y" ;\n"
#define XO4(x,y) " xorps "OFFS(x)"(%5), %%xmm"#y" ;\n"
#define XO5(x,y) " xorps "OFFS(x)"(%6), %%xmm"#y" ;\n"
#define PF0(x) " prefetchnta "PF_OFFS(x)"(%[p1]) ;\n"
#define LD(x,y) " movaps "OFFS(x)"(%[p1]), %%xmm"#y" ;\n"
#define ST(x,y) " movaps %%xmm"#y", "OFFS(x)"(%[p1]) ;\n"
#define PF1(x) " prefetchnta "PF_OFFS(x)"(%[p2]) ;\n"
#define PF2(x) " prefetchnta "PF_OFFS(x)"(%[p3]) ;\n"
#define PF3(x) " prefetchnta "PF_OFFS(x)"(%[p4]) ;\n"
#define PF4(x) " prefetchnta "PF_OFFS(x)"(%[p5]) ;\n"
#define PF5(x) " prefetchnta "PF_OFFS(x)"(%[p6]) ;\n"
#define XO1(x,y) " xorps "OFFS(x)"(%[p2]), %%xmm"#y" ;\n"
#define XO2(x,y) " xorps "OFFS(x)"(%[p3]), %%xmm"#y" ;\n"
#define XO3(x,y) " xorps "OFFS(x)"(%[p4]), %%xmm"#y" ;\n"
#define XO4(x,y) " xorps "OFFS(x)"(%[p5]), %%xmm"#y" ;\n"
#define XO5(x,y) " xorps "OFFS(x)"(%[p6]), %%xmm"#y" ;\n"
static void
xor_sse_2(unsigned long bytes, unsigned long *p1, unsigned long *p2)
{
unsigned long lines = bytes >> 8;
char xmm_save[16*4];
unsigned int lines = bytes >> 8;
unsigned long cr0;
xmm_store_t xmm_save[4];
XMMS_SAVE;
__asm__ __volatile__ (
asm volatile (
#undef BLOCK
#define BLOCK(i) \
LD(i,0) \
......@@ -612,13 +119,11 @@ xor_sse_2(unsigned long bytes, unsigned long *p1, unsigned long *p2)
BLOCK(8)
BLOCK(12)
" addq $256, %1 ;\n"
" addq $256, %2 ;\n"
" decq %0 ;\n"
" jnz 1b ;\n"
:
: "r" (lines),
"r" (p1), "r" (p2)
" addq %[inc], %[p1] ;\n"
" addq %[inc], %[p2] ;\n"
" decl %[cnt] ; jnz 1b"
: [p1] "+r" (p1), [p2] "+r" (p2), [cnt] "+r" (lines)
: [inc] "r" (256UL)
: "memory");
XMMS_RESTORE;
......@@ -628,8 +133,8 @@ static void
xor_sse_3(unsigned long bytes, unsigned long *p1, unsigned long *p2,
unsigned long *p3)
{
unsigned long lines = bytes >> 8;
char xmm_save[16*4];
unsigned int lines = bytes >> 8;
xmm_store_t xmm_save[4];
unsigned long cr0;
XMMS_SAVE;
......@@ -672,16 +177,14 @@ xor_sse_3(unsigned long bytes, unsigned long *p1, unsigned long *p2,
BLOCK(8)
BLOCK(12)
" addq $256, %1 ;\n"
" addq $256, %2 ;\n"
" addq $256, %3 ;\n"
" decq %0 ;\n"
" jnz 1b ;\n"
:
: "r" (lines),
"r" (p1), "r"(p2), "r"(p3)
: "memory" );
" addq %[inc], %[p1] ;\n"
" addq %[inc], %[p2] ;\n"
" addq %[inc], %[p3] ;\n"
" decl %[cnt] ; jnz 1b"
: [cnt] "+r" (lines),
[p1] "+r" (p1), [p2] "+r" (p2), [p3] "+r" (p3)
: [inc] "r" (256UL)
: "memory");
XMMS_RESTORE;
}
......@@ -689,8 +192,8 @@ static void
xor_sse_4(unsigned long bytes, unsigned long *p1, unsigned long *p2,
unsigned long *p3, unsigned long *p4)
{
unsigned long lines = bytes >> 8;
char xmm_save[16*4];
unsigned int lines = bytes >> 8;
xmm_store_t xmm_save[4];
unsigned long cr0;
XMMS_SAVE;
......@@ -739,15 +242,14 @@ xor_sse_4(unsigned long bytes, unsigned long *p1, unsigned long *p2,
BLOCK(8)
BLOCK(12)
" addq $256, %1 ;\n"
" addq $256, %2 ;\n"
" addq $256, %3 ;\n"
" addq $256, %4 ;\n"
" decq %0 ;\n"
" jnz 1b ;\n"
:
: "r" (lines),
"r" (p1), "r" (p2), "r" (p3), "r" (p4)
" addq %[inc], %[p1] ;\n"
" addq %[inc], %[p2] ;\n"
" addq %[inc], %[p3] ;\n"
" addq %[inc], %[p4] ;\n"
" decl %[cnt] ; jnz 1b"
: [cnt] "+c" (lines),
[p1] "+r" (p1), [p2] "+r" (p2), [p3] "+r" (p3), [p4] "+r" (p4)
: [inc] "r" (256UL)
: "memory" );
XMMS_RESTORE;
......@@ -757,8 +259,8 @@ static void
xor_sse_5(unsigned long bytes, unsigned long *p1, unsigned long *p2,
unsigned long *p3, unsigned long *p4, unsigned long *p5)
{
unsigned long lines = bytes >> 8;
char xmm_save[16*4];
unsigned int lines = bytes >> 8;
xmm_store_t xmm_save[4];
unsigned long cr0;
XMMS_SAVE;
......@@ -813,47 +315,36 @@ xor_sse_5(unsigned long bytes, unsigned long *p1, unsigned long *p2,
BLOCK(8)
BLOCK(12)
" addq $256, %1 ;\n"
" addq $256, %2 ;\n"
" addq $256, %3 ;\n"
" addq $256, %4 ;\n"
" addq $256, %5 ;\n"
" decq %0 ;\n"
" jnz 1b ;\n"
:
: "r" (lines),
"r" (p1), "r" (p2), "r" (p3), "r" (p4), "r" (p5)
" addq %[inc], %[p1] ;\n"
" addq %[inc], %[p2] ;\n"
" addq %[inc], %[p3] ;\n"
" addq %[inc], %[p4] ;\n"
" addq %[inc], %[p5] ;\n"
" decl %[cnt] ; jnz 1b"
: [cnt] "+c" (lines),
[p1] "+r" (p1), [p2] "+r" (p2), [p3] "+r" (p3), [p4] "+r" (p4),
[p5] "+r" (p5)
: [inc] "r" (256UL)
: "memory");
XMMS_RESTORE;
}
static struct xor_block_template xor_block_pIII_sse = {
name: "pIII_sse",
static struct xor_block_template xor_block_sse = {
name: "generic_sse",
do_2: xor_sse_2,
do_3: xor_sse_3,
do_4: xor_sse_4,
do_5: xor_sse_5,
};
/* Also try the generic routines. */
#include <asm-generic/xor.h>
#undef XOR_TRY_TEMPLATES
#define XOR_TRY_TEMPLATES \
do { \
xor_speed(&xor_block_8regs); \
xor_speed(&xor_block_32regs); \
if (cpu_has_xmm) \
xor_speed(&xor_block_pIII_sse); \
if (md_cpu_has_mmx()) { \
xor_speed(&xor_block_pII_mmx); \
xor_speed(&xor_block_p5_mmx); \
} \
xor_speed(&xor_block_sse); \
} while (0)
/* We force the use of the SSE xor block because it can write around L2.
We may also be able to load into the L1 only depending on how the cpu
deals with a load to a line that is being prefetched. */
#define XOR_SELECT_TEMPLATE(FASTEST) \
(cpu_has_xmm ? &xor_block_pIII_sse : FASTEST)
#define XOR_SELECT_TEMPLATE(FASTEST) (&xor_block_sse)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7