misc: mic: Update MIC host daemon with COSM changes

This patch updates the MIC host daemon to work with corresponding
changes in COSM. Other MIC daemon fixes, cleanups and enhancements as
are also rolled into this patch. Changes to MIC sysfs ABI which go
into effect with this patch are also documented.
Reviewed-by: Sudeep Dutt <sudeep.dutt@intel.com>
Signed-off-by: Dasaratharaman Chandramouli <dasaratharaman.chandramouli@intel.com>
Signed-off-by: Ashutosh Dixit <ashutosh.dixit@intel.com>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>

Documentation/ABI/testing/sysfs-class-mic.txt

@@ -41,18 +41,15 @@ Description:
 		When read, this entry provides the current state of an Intel
 		MIC device in the context of the card OS. Possible values that
 		will be read are:
-		"offline" - The MIC device is ready to boot the card OS. On
+		"ready" - The MIC device is ready to boot the card OS. On
 		reading this entry after an OSPM resume, a "boot" has to be
 		written to this entry if the card was previously shutdown
 		during OSPM suspend.
-		"online" - The MIC device has initiated booting a card OS.
+		"booting" - The MIC device has initiated booting a card OS.
+		"online" - The MIC device has completed boot and is online
 		"shutting_down" - The card OS is shutting down.
+		"resetting" - A reset has been initiated for the MIC device
 		"reset_failed" - The MIC device has failed to reset.
-		"suspending" - The MIC device is currently being prepared for
-		suspend. On reading this entry, a "suspend" has to be written
-		to the state sysfs entry to ensure the card is shutdown during
-		OSPM suspend.
-		"suspended" - The MIC device has been suspended.
 
 		When written, this sysfs entry triggers different state change
 		operations depending upon the current state of the card OS.
@@ -62,8 +59,6 @@ Description:
 			sysfs entries.
 		"reset" - Initiates device reset.
 		"shutdown" - Initiates card OS shutdown.
-		"suspend" - Initiates card OS shutdown and also marks the card
-		as suspended.
 
 What:		/sys/class/mic/mic(x)/shutdown_status
 Date:		October 2013
@@ -126,7 +121,7 @@ Description:
 		the card. This sysfs entry can be written with the following
 		valid strings:
 		a) linux - Boot a Linux image.
-		b) elf - Boot an elf image for flash updates.
+		b) flash - Boot an image for flash updates.
 
 What:		/sys/class/mic/mic(x)/log_buf_addr
 Date:		October 2013
@@ -155,3 +150,17 @@ Description:
 		daemon to set the log buffer length address. The correct log
 		buffer length address to be written can be found in the
 		System.map file of the card OS.
+
+What:		/sys/class/mic/mic(x)/heartbeat_enable
+Date:		March 2015
+KernelVersion:	3.20
+Contact:	Ashutosh Dixit <ashutosh.dixit@intel.com>
+Description:
+		The MIC drivers detect and inform user space about card crashes
+		via a heartbeat mechanism (see the description of
+		shutdown_status above). User space can turn off this
+		notification by setting heartbeat_enable to 0 and enable it by
+		setting this entry to 1. If this notification is disabled it is
+		the responsibility of user space to detect card crashes via
+		alternative means such as a network ping. This setting is
+		enabled by default.

Documentation/mic/mic_overview.txt

@@ -28,6 +28,10 @@ The Symmetric Communication Interface (SCIF (pronounced as skiff)) is a
 low level communications API across PCIe currently implemented for MIC.
 More details are available at scif_overview.txt.
 
+The Coprocessor State Management (COSM) driver on the host allows for
+boot, shutdown and reset of Intel MIC devices. It communicates with a COSM
+"client" driver on the MIC cards over SCIF to perform these functions.
+
 Here is a block diagram of the various components described above. The
 virtio backends are situated on the host rather than the card given better
 single threaded performance for the host compared to MIC, the ability of
@@ -51,19 +55,20 @@ the fact that the virtio block storage backend can only be on the host.
                       |               |       | Virtio over PCIe IOCTLs  |
                       |               |       +--------------------------+
 +-----------+         |               |                   |  +-----------+
-| MIC DMA   |         |  +----------+ | +-----------+     |  | MIC DMA   |
-| Driver    |         |  |  SCIF    | | |   SCIF    |     |  | Driver    |
-+-----------+         |  +----------+ | +-----------+     |  +-----------+
-      |               |       |       |       |           |        |
-+---------------+     | +-----+-----+ | +-----+-----+     | +---------------+
-|MIC virtual Bus|     | |SCIF HW Bus| | |SCIF HW BUS|     | |MIC virtual Bus|
-+---------------+     | +-----------+ | +-----+-----+     | +---------------+
-      |               |       |       |       |           |              |
-      |   +--------------+    |       |       |    +---------------+     |
-      |   |Intel MIC     |    |       |       |    |Intel MIC      |     |
-      +---|Card Driver   +----+       |       |    |Host Driver    |     |
-          +--------------+            |       +----+---------------+-----+
-                      |               |                   |
+| MIC DMA   |         |      +------+ | +------+ +------+ |  | MIC DMA   |
+| Driver    |         |      | SCIF | | | SCIF | | COSM | |  | Driver    |
++-----------+         |      +------+ | +------+ +--+---+ |  +-----------+
+      |               |         |     |    |        |     |        |
++---------------+     |      +------+ | +--+---+ +--+---+ | +----------------+
+|MIC virtual Bus|     |      |SCIF  | | |SCIF  | | COSM | | |MIC virtual Bus |
++---------------+     |      |HW Bus| | |HW Bus| | Bus  | | +----------------+
+      |               |      +------+ | +--+---+ +------+ |              |
+      |               |         |     |       |     |     |              |
+      |   +-----------+---+     |     |       |    +---------------+     |
+      |   |Intel MIC      |     |     |       |    |Intel MIC      |     |
+      +---|Card Driver    |     |     |       |    |Host Driver    |     |
+          +------------+--------+     |       +----+---------------+-----+
+                 |                    |                   |
              +-------------------------------------------------------------+
              |                                                             |
              |                    PCIe Bus                                 |

Documentation/mic/mpssd/mpss

@@ -119,10 +119,10 @@ stop()
 	# Wait for the cards to go offline
 	for f in $sysfs/*
 	do
-		while [ "`cat $f/state`" != "offline" ]
+		while [ "`cat $f/state`" != "ready" ]
 		do
 			sleep 1
-			echo -e "Waiting for "`basename $f`" to go offline"
+			echo -e "Waiting for "`basename $f`" to become ready"
 		done
 	done
 

Documentation/mic/mpssd/mpssd.c

@@ -43,7 +43,7 @@
 #include <linux/mic_common.h>
 #include <tools/endian.h>
 
-static void init_mic(struct mic_info *mic);
+static void *init_mic(void *arg);
 
 static FILE *logfp;
 static struct mic_info mic_list;
@@ -116,19 +116,18 @@ static struct {
 		.num = htole16(MIC_VRING_ENTRIES),
 	},
 #if GSO_ENABLED
-		.host_features = htole32(
+	.host_features = htole32(
 		1 << VIRTIO_NET_F_CSUM |
 		1 << VIRTIO_NET_F_GSO |
 		1 << VIRTIO_NET_F_GUEST_TSO4 |
 		1 << VIRTIO_NET_F_GUEST_TSO6 |
-		1 << VIRTIO_NET_F_GUEST_ECN |
-		1 << VIRTIO_NET_F_GUEST_UFO),
+		1 << VIRTIO_NET_F_GUEST_ECN),
 #else
 		.host_features = 0,
 #endif
 };
 
-static const char *mic_config_dir = "/etc/sysconfig/mic";
+static const char *mic_config_dir = "/etc/mpss";
 static const char *virtblk_backend = "VIRTBLK_BACKEND";
 static struct {
 	struct mic_device_desc dd;
@@ -192,7 +191,7 @@ tap_configure(struct mic_info *mic, char *dev)
 		return ret;
 	}
 
-	snprintf(ipaddr, IFNAMSIZ, "172.31.%d.254/24", mic->id);
+	snprintf(ipaddr, IFNAMSIZ, "172.31.%d.254/24", mic->id + 1);
 
 	pid = fork();
 	if (pid == 0) {
@@ -255,8 +254,7 @@ static int tun_alloc(struct mic_info *mic, char *dev)
 		return err;
 	}
 #if GSO_ENABLED
-	offload = TUN_F_CSUM | TUN_F_TSO4 | TUN_F_TSO6 |
-		TUN_F_TSO_ECN | TUN_F_UFO;
+	offload = TUN_F_CSUM | TUN_F_TSO4 | TUN_F_TSO6 | TUN_F_TSO_ECN;
 
 	err = ioctl(fd, TUNSETOFFLOAD, offload);
 	if (err < 0) {
@@ -332,7 +330,6 @@ static struct mic_device_desc *get_device_desc(struct mic_info *mic, int type)
 			return d;
 	}
 	mpsslog("%s %s %d not found\n", mic->name, __func__, type);
-	assert(0);
 	return NULL;
 }
 
@@ -415,6 +412,13 @@ mic_virtio_copy(struct mic_info *mic, int fd,
 	return ret;
 }
 
+static inline unsigned _vring_size(unsigned int num, unsigned long align)
+{
+	return ((sizeof(struct vring_desc) * num + sizeof(__u16) * (3 + num)
+				+ align - 1) & ~(align - 1))
+		+ sizeof(__u16) * 3 + sizeof(struct vring_used_elem) * num;
+}
+
 /*
  * This initialization routine requires at least one
  * vring i.e. vr0. vr1 is optional.
@@ -426,8 +430,9 @@ init_vr(struct mic_info *mic, int fd, int type,
 	int vr_size;
 	char *va;
 
-	vr_size = PAGE_ALIGN(vring_size(MIC_VRING_ENTRIES,
-		MIC_VIRTIO_RING_ALIGN) + sizeof(struct _mic_vring_info));
+	vr_size = PAGE_ALIGN(_vring_size(MIC_VRING_ENTRIES,
+					 MIC_VIRTIO_RING_ALIGN) +
+			     sizeof(struct _mic_vring_info));
 	va = mmap(NULL, MIC_DEVICE_PAGE_END + vr_size * num_vq,
 		PROT_READ, MAP_SHARED, fd, 0);
 	if (MAP_FAILED == va) {
@@ -439,25 +444,25 @@ init_vr(struct mic_info *mic, int fd, int type,
 	set_dp(mic, type, va);
 	vr0->va = (struct mic_vring *)&va[MIC_DEVICE_PAGE_END];
 	vr0->info = vr0->va +
-		vring_size(MIC_VRING_ENTRIES, MIC_VIRTIO_RING_ALIGN);
+		_vring_size(MIC_VRING_ENTRIES, MIC_VIRTIO_RING_ALIGN);
 	vring_init(&vr0->vr,
 		   MIC_VRING_ENTRIES, vr0->va, MIC_VIRTIO_RING_ALIGN);
 	mpsslog("%s %s vr0 %p vr0->info %p vr_size 0x%x vring 0x%x ",
 		__func__, mic->name, vr0->va, vr0->info, vr_size,
-		vring_size(MIC_VRING_ENTRIES, MIC_VIRTIO_RING_ALIGN));
+		_vring_size(MIC_VRING_ENTRIES, MIC_VIRTIO_RING_ALIGN));
 	mpsslog("magic 0x%x expected 0x%x\n",
 		le32toh(vr0->info->magic), MIC_MAGIC + type);
 	assert(le32toh(vr0->info->magic) == MIC_MAGIC + type);
 	if (vr1) {
 		vr1->va = (struct mic_vring *)
 			&va[MIC_DEVICE_PAGE_END + vr_size];
-		vr1->info = vr1->va + vring_size(MIC_VRING_ENTRIES,
+		vr1->info = vr1->va + _vring_size(MIC_VRING_ENTRIES,
 			MIC_VIRTIO_RING_ALIGN);
 		vring_init(&vr1->vr,
 			   MIC_VRING_ENTRIES, vr1->va, MIC_VIRTIO_RING_ALIGN);
 		mpsslog("%s %s vr1 %p vr1->info %p vr_size 0x%x vring 0x%x ",
 			__func__, mic->name, vr1->va, vr1->info, vr_size,
-			vring_size(MIC_VRING_ENTRIES, MIC_VIRTIO_RING_ALIGN));
+			_vring_size(MIC_VRING_ENTRIES, MIC_VIRTIO_RING_ALIGN));
 		mpsslog("magic 0x%x expected 0x%x\n",
 			le32toh(vr1->info->magic), MIC_MAGIC + type + 1);
 		assert(le32toh(vr1->info->magic) == MIC_MAGIC + type + 1);
@@ -466,16 +471,21 @@ init_vr(struct mic_info *mic, int fd, int type,
 	return va;
 }
 
-static void
+static int
 wait_for_card_driver(struct mic_info *mic, int fd, int type)
 {
 	struct pollfd pollfd;
 	int err;
 	struct mic_device_desc *desc = get_device_desc(mic, type);
+	__u8 prev_status;
 
+	if (!desc)
+		return -ENODEV;
+	prev_status = desc->status;
 	pollfd.fd = fd;
 	mpsslog("%s %s Waiting .... desc-> type %d status 0x%x\n",
 		mic->name, __func__, type, desc->status);
+
 	while (1) {
 		pollfd.events = POLLIN;
 		pollfd.revents = 0;
@@ -487,8 +497,13 @@ wait_for_card_driver(struct mic_info *mic, int fd, int type)
 		}
 
 		if (pollfd.revents) {
-			mpsslog("%s %s Waiting... desc-> type %d status 0x%x\n",
-				mic->name, __func__, type, desc->status);
+			if (desc->status != prev_status) {
+				mpsslog("%s %s Waiting... desc-> type %d "
+					"status 0x%x\n",
+					mic->name, __func__, type,
+					desc->status);
+				prev_status = desc->status;
+			}
 			if (desc->status & VIRTIO_CONFIG_S_DRIVER_OK) {
 				mpsslog("%s %s poll.revents %d\n",
 					mic->name, __func__, pollfd.revents);
@@ -499,6 +514,7 @@ wait_for_card_driver(struct mic_info *mic, int fd, int type)
 			}
 		}
 	}
+	return 0;
 }
 
 /* Spin till we have some descriptors */
@@ -575,9 +591,16 @@ virtio_net(void *arg)
 				__func__, strerror(errno));
 			continue;
 		}
-		if (!(desc->status & VIRTIO_CONFIG_S_DRIVER_OK))
-			wait_for_card_driver(mic, mic->mic_net.virtio_net_fd,
-					     VIRTIO_ID_NET);
+		if (!(desc->status & VIRTIO_CONFIG_S_DRIVER_OK)) {
+			err = wait_for_card_driver(mic,
+						   mic->mic_net.virtio_net_fd,
+						   VIRTIO_ID_NET);
+			if (err) {
+				mpsslog("%s %s %d Exiting...\n",
+					mic->name, __func__, __LINE__);
+				break;
+			}
+		}
 		/*
 		 * Check if there is data to be read from TUN and write to
 		 * virtio net fd if there is.
@@ -786,10 +809,16 @@ virtio_console(void *arg)
 				strerror(errno));
 			continue;
 		}
-		if (!(desc->status & VIRTIO_CONFIG_S_DRIVER_OK))
-			wait_for_card_driver(mic,
-					     mic->mic_console.virtio_console_fd,
-				VIRTIO_ID_CONSOLE);
+		if (!(desc->status & VIRTIO_CONFIG_S_DRIVER_OK)) {
+			err = wait_for_card_driver(mic,
+					mic->mic_console.virtio_console_fd,
+					VIRTIO_ID_CONSOLE);
+			if (err) {
+				mpsslog("%s %s %d Exiting...\n",
+					mic->name, __func__, __LINE__);
+				break;
+			}
+		}
 
 		if (console_poll[MONITOR_FD].revents & POLLIN) {
 			copy.iov = iov0;
@@ -1048,8 +1077,9 @@ stop_virtblk(struct mic_info *mic)
 {
 	int vr_size, ret;
 
-	vr_size = PAGE_ALIGN(vring_size(MIC_VRING_ENTRIES,
-		MIC_VIRTIO_RING_ALIGN) + sizeof(struct _mic_vring_info));
+	vr_size = PAGE_ALIGN(_vring_size(MIC_VRING_ENTRIES,
+					 MIC_VIRTIO_RING_ALIGN) +
+			     sizeof(struct _mic_vring_info));
 	ret = munmap(mic->mic_virtblk.block_dp,
 		MIC_DEVICE_PAGE_END + vr_size * virtblk_dev_page.dd.num_vq);
 	if (ret < 0)
@@ -1131,6 +1161,10 @@ write_status(int fd, __u8 *status)
 	return ioctl(fd, MIC_VIRTIO_COPY_DESC, &copy);
 }
 
+#ifndef VIRTIO_BLK_T_GET_ID
+#define VIRTIO_BLK_T_GET_ID    8
+#endif
+
 static void *
 virtio_block(void *arg)
 {
@@ -1297,12 +1331,7 @@ reset(struct mic_info *mic)
 		mpsslog("%s: %s %d state %s\n",
 			mic->name, __func__, __LINE__, state);
 
-		/*
-		 * If the shutdown was initiated by OSPM, the state stays
-		 * in "suspended" which is also a valid condition for reset.
-		 */
-		if ((!strcmp(state, "offline")) ||
-		    (!strcmp(state, "suspended"))) {
+		if (!strcmp(state, "ready")) {
 			free(state);
 			break;
 		}
@@ -1331,34 +1360,50 @@ get_mic_shutdown_status(struct mic_info *mic, char *shutdown_status)
 	assert(0);
 };
 
-static int get_mic_state(struct mic_info *mic, char *state)
+static int get_mic_state(struct mic_info *mic)
 {
-	if (!strcmp(state, "offline"))
-		return MIC_OFFLINE;
-	if (!strcmp(state, "online"))
-		return MIC_ONLINE;
-	if (!strcmp(state, "shutting_down"))
-		return MIC_SHUTTING_DOWN;
-	if (!strcmp(state, "reset_failed"))
-		return MIC_RESET_FAILED;
-	if (!strcmp(state, "suspending"))
-		return MIC_SUSPENDING;
-	if (!strcmp(state, "suspended"))
-		return MIC_SUSPENDED;
-	mpsslog("%s: BUG invalid state %s\n", mic->name, state);
-	/* Invalid state */
-	assert(0);
+	char *state = NULL;
+	enum mic_states mic_state;
+
+	while (!state) {
+		state = readsysfs(mic->name, "state");
+		sleep(1);
+	}
+	mpsslog("%s: %s %d state %s\n",
+		mic->name, __func__, __LINE__, state);
+
+	if (!strcmp(state, "ready")) {
+		mic_state = MIC_READY;
+	} else if (!strcmp(state, "booting")) {
+		mic_state = MIC_BOOTING;
+	} else if (!strcmp(state, "online")) {
+		mic_state = MIC_ONLINE;
+	} else if (!strcmp(state, "shutting_down")) {
+		mic_state = MIC_SHUTTING_DOWN;
+	} else if (!strcmp(state, "reset_failed")) {
+		mic_state = MIC_RESET_FAILED;
+	} else if (!strcmp(state, "resetting")) {
+		mic_state = MIC_RESETTING;
+	} else {
+		mpsslog("%s: BUG invalid state %s\n", mic->name, state);
+		assert(0);
+	}
+
+	free(state);
+	return mic_state;
 };
 
 static void mic_handle_shutdown(struct mic_info *mic)
 {
 #define SHUTDOWN_TIMEOUT 60
-	int i = SHUTDOWN_TIMEOUT, ret, stat = 0;
+	int i = SHUTDOWN_TIMEOUT;
 	char *shutdown_status;
 	while (i) {
 		shutdown_status = readsysfs(mic->name, "shutdown_status");
-		if (!shutdown_status)
+		if (!shutdown_status) {
+			sleep(1);
 			continue;
+		}
 		mpsslog("%s: %s %d shutdown_status %s\n",
 			mic->name, __func__, __LINE__, shutdown_status);
 		switch (get_mic_shutdown_status(mic, shutdown_status)) {
@@ -1377,94 +1422,110 @@ static void mic_handle_shutdown(struct mic_info *mic)
 		i--;
 	}
 reset:
-	ret = kill(mic->pid, SIGTERM);
-	mpsslog("%s: %s %d kill pid %d ret %d\n",
-		mic->name, __func__, __LINE__,
-		mic->pid, ret);
-	if (!ret) {
-		ret = waitpid(mic->pid, &stat,
-			WIFSIGNALED(stat));
-		mpsslog("%s: %s %d waitpid ret %d pid %d\n",
-			mic->name, __func__, __LINE__,
-			ret, mic->pid);
-	}
-	if (ret == mic->pid)
-		reset(mic);
+	if (!i)
+		mpsslog("%s: %s %d timing out waiting for shutdown_status %s\n",
+			mic->name, __func__, __LINE__, shutdown_status);
+	reset(mic);
 }
 
-static void *
-mic_config(void *arg)
+static int open_state_fd(struct mic_info *mic)
 {
-	struct mic_info *mic = (struct mic_info *)arg;
-	char *state = NULL;
 	char pathname[PATH_MAX];
-	int fd, ret;
-	struct pollfd ufds[1];
-	char value[4096];
+	int fd;
 
 	snprintf(pathname, PATH_MAX - 1, "%s/%s/%s",
 		 MICSYSFSDIR, mic->name, "state");
 
 	fd = open(pathname, O_RDONLY);
-	if (fd < 0) {
+	if (fd < 0)
 		mpsslog("%s: opening file %s failed %s\n",
 			mic->name, pathname, strerror(errno));
-		goto error;
+	return fd;
+}
+
+static int block_till_state_change(int fd, struct mic_info *mic)
+{
+	struct pollfd ufds[1];
+	char value[PAGE_SIZE];
+	int ret;
+
+	ufds[0].fd = fd;
+	ufds[0].events = POLLERR | POLLPRI;
+	ret = poll(ufds, 1, -1);
+	if (ret < 0) {
+		mpsslog("%s: %s %d poll failed %s\n",
+			mic->name, __func__, __LINE__, strerror(errno));
+		return ret;
+	}
+
+	ret = lseek(fd, 0, SEEK_SET);
+	if (ret < 0) {
+		mpsslog("%s: %s %d Failed to seek to 0: %s\n",
+			mic->name, __func__, __LINE__, strerror(errno));
+		return ret;
+	}
+
+	ret = read(fd, value, sizeof(value));
+	if (ret < 0) {
+		mpsslog("%s: %s %d Failed to read sysfs entry: %s\n",
+			mic->name, __func__, __LINE__, strerror(errno));
+		return ret;
+	}
+
+	return 0;
+}
+
+static void *
+mic_config(void *arg)
+{
+	struct mic_info *mic = (struct mic_info *)arg;
+	int fd, ret, stat = 0;
+
+	fd = open_state_fd(mic);
+	if (fd < 0) {
+		mpsslog("%s: %s %d open state fd failed %s\n",
+			mic->name, __func__, __LINE__, strerror(errno));
+		goto exit;
 	}
 
 	do {
-		ret = lseek(fd, 0, SEEK_SET);
+		ret = block_till_state_change(fd, mic);
 		if (ret < 0) {
-			mpsslog("%s: Failed to seek to file start '%s': %s\n",
-				mic->name, pathname, strerror(errno));
-			goto close_error1;
+			mpsslog("%s: %s %d block_till_state_change error %s\n",
+				mic->name, __func__, __LINE__, strerror(errno));
+			goto close_exit;
 		}
-		ret = read(fd, value, sizeof(value));
-		if (ret < 0) {
-			mpsslog("%s: Failed to read sysfs entry '%s': %s\n",
-				mic->name, pathname, strerror(errno));
-			goto close_error1;
-		}
-retry:
-		state = readsysfs(mic->name, "state");
-		if (!state)
-			goto retry;
-		mpsslog("%s: %s %d state %s\n",
-			mic->name, __func__, __LINE__, state);
-		switch (get_mic_state(mic, state)) {
+
+		switch (get_mic_state(mic)) {
 		case MIC_SHUTTING_DOWN:
 			mic_handle_shutdown(mic);
-			goto close_error;
-		case MIC_SUSPENDING:
-			mic->boot_on_resume = 1;
-			setsysfs(mic->name, "state", "suspend");
-			mic_handle_shutdown(mic);
-			goto close_error;
-		case MIC_OFFLINE:
+			break;
+		case MIC_READY:
+		case MIC_RESET_FAILED:
+			ret = kill(mic->pid, SIGTERM);
+			mpsslog("%s: %s %d kill pid %d ret %d\n",
+				mic->name, __func__, __LINE__,
+				mic->pid, ret);
+			if (!ret) {
+				ret = waitpid(mic->pid, &stat,
+					      WIFSIGNALED(stat));
+				mpsslog("%s: %s %d waitpid ret %d pid %d\n",
+					mic->name, __func__, __LINE__,
+					ret, mic->pid);
+			}
 			if (mic->boot_on_resume) {
 				setsysfs(mic->name, "state", "boot");
 				mic->boot_on_resume = 0;
 			}
-			break;
+			goto close_exit;
 		default:
 			break;
 		}
-		free(state);
-
-		ufds[0].fd = fd;
-		ufds[0].events = POLLERR | POLLPRI;
-		ret = poll(ufds, 1, -1);
-		if (ret < 0) {
-			mpsslog("%s: poll failed %s\n",
-				mic->name, strerror(errno));
-			goto close_error1;
-		}
 	} while (1);
-close_error:
-	free(state);
-close_error1:
+
+close_exit:
 	close(fd);
-error:
+exit:
 	init_mic(mic);
 	pthread_exit(NULL);
 }
@@ -1477,15 +1538,15 @@ set_cmdline(struct mic_info *mic)
 
 	len = snprintf(buffer, PATH_MAX,
 		"clocksource=tsc highres=off nohz=off ");
-	len += snprintf(buffer + len, PATH_MAX - len,
+	len += snprintf(buffer + len, PATH_MAX,
 		"cpufreq_on;corec6_off;pc3_off;pc6_off ");
-	len += snprintf(buffer + len, PATH_MAX - len,
+	len += snprintf(buffer + len, PATH_MAX,
 		"ifcfg=static;address,172.31.%d.1;netmask,255.255.255.0",
-		mic->id);
+		mic->id + 1);
 
 	setsysfs(mic->name, "cmdline", buffer);
 	mpsslog("%s: Command line: \"%s\"\n", mic->name, buffer);
-	snprintf(buffer, PATH_MAX, "172.31.%d.1", mic->id);
+	snprintf(buffer, PATH_MAX, "172.31.%d.1", mic->id + 1);
 	mpsslog("%s: IPADDR: \"%s\"\n", mic->name, buffer);
 }
 
@@ -1541,8 +1602,6 @@ set_log_buf_info(struct mic_info *mic)
 	close(fd);
 }
 
-static void init_mic(struct mic_info *mic);
-
 static void
 change_virtblk_backend(int x, siginfo_t *siginfo, void *p)
 {
@@ -1553,8 +1612,16 @@ change_virtblk_backend(int x, siginfo_t *siginfo, void *p)
 }
 
 static void
-init_mic(struct mic_info *mic)
+set_mic_boot_params(struct mic_info *mic)
+{
+	set_log_buf_info(mic);
+	set_cmdline(mic);
+}
+
+static void *
+init_mic(void *arg)
 {
+	struct mic_info *mic = (struct mic_info *)arg;
 	struct sigaction ignore = {
 		.sa_flags = 0,
 		.sa_handler = SIG_IGN
@@ -1564,7 +1631,7 @@ init_mic(struct mic_info *mic)
 		.sa_sigaction = change_virtblk_backend,
 	};
 	char buffer[PATH_MAX];
-	int err;
+	int err, fd;
 
 	/*
 	 * Currently, one virtio block device is supported for each MIC card
@@ -1577,12 +1644,38 @@ init_mic(struct mic_info *mic)
 	 * the MIC daemon.
 	 */
 	sigaction(SIGUSR1, &ignore, NULL);
+retry:
+	fd = open_state_fd(mic);
+	if (fd < 0) {
+		mpsslog("%s: %s %d open state fd failed %s\n",
+			mic->name, __func__, __LINE__, strerror(errno));
+		sleep(2);
+		goto retry;
+	}
+
+	if (mic->restart) {
+		snprintf(buffer, PATH_MAX, "boot");
+		setsysfs(mic->name, "state", buffer);
+		mpsslog("%s restarting mic %d\n",
+			mic->name, mic->restart);
+		mic->restart = 0;
+	}
+
+	while (1) {
+		while (block_till_state_change(fd, mic)) {
+			mpsslog("%s: %s %d block_till_state_change error %s\n",
+				mic->name, __func__, __LINE__, strerror(errno));
+			sleep(2);
+			continue;
+		}
+
+		if (get_mic_state(mic) == MIC_BOOTING)
+			break;
+	}
 
 	mic->pid = fork();
 	switch (mic->pid) {
 	case 0:
-		set_log_buf_info(mic);
-		set_cmdline(mic);
 		add_virtio_device(mic, &virtcons_dev_page.dd);
 		add_virtio_device(mic, &virtnet_dev_page.dd);
 		err = pthread_create(&mic->mic_console.console_thread, NULL,
@@ -1612,24 +1705,29 @@ init_mic(struct mic_info *mic)
 			mic->name, mic->id, errno);
 		break;
 	default:
-		if (mic->restart) {
-			snprintf(buffer, PATH_MAX, "boot");
-			setsysfs(mic->name, "state", buffer);
-			mpsslog("%s restarting mic %d\n",
-				mic->name, mic->restart);
-			mic->restart = 0;
-		}
-		pthread_create(&mic->config_thread, NULL, mic_config, mic);
+		err = pthread_create(&mic->config_thread, NULL,
+				     mic_config, mic);
+		if (err)
+			mpsslog("%s mic_config pthread_create failed %s\n",
+				mic->name, strerror(err));
 	}
+
+	return NULL;
 }
 
 static void
 start_daemon(void)
 {
 	struct mic_info *mic;
+	int err;
 
-	for (mic = mic_list.next; mic != NULL; mic = mic->next)
-		init_mic(mic);
+	for (mic = mic_list.next; mic; mic = mic->next) {
+		set_mic_boot_params(mic);
+		err = pthread_create(&mic->init_thread, NULL, init_mic, mic);
+		if (err)
+			mpsslog("%s init_mic pthread_create failed %s\n",
+				mic->name, strerror(err));
+	}
 
 	while (1)
 		sleep(60);

Documentation/mic/mpssd/mpssd.h

@@ -86,6 +86,7 @@ struct mic_info {
 	int		id;
 	char		*name;
 	pthread_t       config_thread;
+	pthread_t       init_thread;
 	pid_t		pid;
 	struct mic_console_info	mic_console;
 	struct mic_net_info	mic_net;

drivers/misc/mic/Makefile

@@ -2,5 +2,9 @@
 # Makefile - Intel MIC Linux driver.
 # Copyright(c) 2013, Intel Corporation.
 #
+obj-$(CONFIG_INTEL_MIC_HOST) += host/
+obj-$(CONFIG_INTEL_MIC_CARD) += card/
 obj-y += bus/
 obj-$(CONFIG_SCIF) += scif/
+obj-$(CONFIG_MIC_COSM) += cosm/
+obj-$(CONFIG_MIC_COSM) += cosm_client/

drivers/misc/mic/bus/scif_bus.c

@@ -28,7 +28,6 @@ static ssize_t device_show(struct device *d,
 
 	return sprintf(buf, "0x%04x\n", dev->id.device);
 }
-
 static DEVICE_ATTR_RO(device);
 
 static ssize_t vendor_show(struct device *d,
@@ -38,7 +37,6 @@ static ssize_t vendor_show(struct device *d,
 
 	return sprintf(buf, "0x%04x\n", dev->id.vendor);
 }
-
 static DEVICE_ATTR_RO(vendor);
 
 static ssize_t modalias_show(struct device *d,
@@ -49,7 +47,6 @@ static ssize_t modalias_show(struct device *d,
 	return sprintf(buf, "scif:d%08Xv%08X\n",
 		       dev->id.device, dev->id.vendor);
 }
-
 static DEVICE_ATTR_RO(modalias);
 
 static struct attribute *scif_dev_attrs[] = {
@@ -144,7 +141,8 @@ struct scif_hw_dev *
 scif_register_device(struct device *pdev, int id, struct dma_map_ops *dma_ops,
 		     struct scif_hw_ops *hw_ops, u8 dnode, u8 snode,
 		     struct mic_mw *mmio, struct mic_mw *aper, void *dp,
-		     void __iomem *rdp, struct dma_chan **chan, int num_chan)
+		     void __iomem *rdp, struct dma_chan **chan, int num_chan,
+		     bool card_rel_da)
 {
 	int ret;
 	struct scif_hw_dev *sdev;
@@ -171,6 +169,7 @@ scif_register_device(struct device *pdev, int id, struct dma_map_ops *dma_ops,
 	dma_set_mask(&sdev->dev, DMA_BIT_MASK(64));
 	sdev->dma_ch = chan;
 	sdev->num_dma_ch = num_chan;
+	sdev->card_rel_da = card_rel_da;
 	dev_set_name(&sdev->dev, "scif-dev%u", sdev->dnode);
 	/*
 	 * device_register() causes the bus infrastructure to look for a

drivers/misc/mic/bus/scif_bus.h

@@ -46,6 +46,8 @@ struct scif_hw_dev_id {
  * @rdp - Remote device page
  * @dma_ch - Array of DMA channels
  * @num_dma_ch - Number of DMA channels available
+ * @card_rel_da - Set to true if DMA addresses programmed in the DMA engine
+ *		are relative to the card point of view
  */
 struct scif_hw_dev {
 	struct scif_hw_ops *hw_ops;
@@ -59,6 +61,7 @@ struct scif_hw_dev {
 	void __iomem *rdp;
 	struct dma_chan **dma_ch;
 	int num_dma_ch;
+	bool card_rel_da;
 };
 
 /**
@@ -114,7 +117,8 @@ scif_register_device(struct device *pdev, int id,
 		     struct scif_hw_ops *hw_ops, u8 dnode, u8 snode,
 		     struct mic_mw *mmio, struct mic_mw *aper,
 		     void *dp, void __iomem *rdp,
-		     struct dma_chan **chan, int num_chan);
+		     struct dma_chan **chan, int num_chan,
+		     bool card_rel_da);
 void scif_unregister_device(struct scif_hw_dev *sdev);
 
 static inline struct scif_hw_dev *dev_to_scif(struct device *dev)

include/uapi/linux/mic_common.h

@@ -75,12 +75,7 @@ struct mic_device_ctrl {
  * struct mic_bootparam: Virtio device independent information in device page
  *
  * @magic: A magic value used by the card to ensure it can see the host
- * @c2h_shutdown_db: Card to Host shutdown doorbell set by host
- * @h2c_shutdown_db: Host to Card shutdown doorbell set by card
  * @h2c_config_db: Host to Card Virtio config doorbell set by card
- * @shutdown_status: Card shutdown status set by card
- * @shutdown_card: Set to 1 by the host when a card shutdown is initiated
- * @tot_nodes: Total number of nodes in the SCIF network
  * @node_id: Unique id of the node
  * @h2c_scif_db - Host to card SCIF doorbell set by card
  * @c2h_scif_db - Card to host SCIF doorbell set by host
@@ -89,12 +84,7 @@ struct mic_device_ctrl {
  */
 struct mic_bootparam {
 	__le32 magic;
-	__s8 c2h_shutdown_db;
-	__s8 h2c_shutdown_db;
 	__s8 h2c_config_db;
-	__u8 shutdown_status;
-	__u8 shutdown_card;
-	__u8 tot_nodes;
 	__u8 node_id;
 	__u8 h2c_scif_db;
 	__u8 c2h_scif_db;
@@ -219,12 +209,12 @@ static inline unsigned mic_total_desc_size(struct mic_device_desc *desc)
  * enum mic_states - MIC states.
  */
 enum mic_states {
-	MIC_OFFLINE = 0,
+	MIC_READY = 0,
+	MIC_BOOTING,
 	MIC_ONLINE,
 	MIC_SHUTTING_DOWN,
+	MIC_RESETTING,
 	MIC_RESET_FAILED,
-	MIC_SUSPENDING,
-	MIC_SUSPENDED,
 	MIC_LAST
 };