[PATCH] /dev/ptmx open() fixes

If tty_open() fails for a normal serial device, we end up doing cleanups
that should only happen for failed open of /dev/ptmx.  The results are
not pretty - devpts et.al.  end up very confused.  That's what gave
problems with ptmx.

This splits ptmx file_operations from the normal case and cleans up both
tty_open() and (new) ptmx_open().  Survived serious beating.

drivers/char/tty_io.c

@@ -142,6 +142,7 @@ static ssize_t tty_write(struct file *, const char __user *, size_t, loff_t *);
 ssize_t redirected_tty_write(struct file *, const char __user *, size_t, loff_t *);
 static unsigned int tty_poll(struct file *, poll_table *);
 static int tty_open(struct inode *, struct file *);
+static int ptmx_open(struct inode *, struct file *);
 static int tty_release(struct inode *, struct file *);
 int tty_ioctl(struct inode * inode, struct file * file,
 	      unsigned int cmd, unsigned long arg);
@@ -377,6 +378,19 @@ static struct file_operations tty_fops = {
 	.fasync		= tty_fasync,
 };
 
+#ifdef CONFIG_UNIX98_PTYS
+static struct file_operations ptmx_fops = {
+	.llseek		= no_llseek,
+	.read		= tty_read,
+	.write		= tty_write,
+	.poll		= tty_poll,
+	.ioctl		= tty_ioctl,
+	.open		= ptmx_open,
+	.release	= tty_release,
+	.fasync		= tty_fasync,
+};
+#endif
+
 static struct file_operations console_fops = {
 	.llseek		= no_llseek,
 	.read		= tty_read,
@@ -1358,45 +1372,6 @@ static int tty_open(struct inode * inode, struct file * filp)
 		return -ENODEV;
 	}
 
-#ifdef CONFIG_UNIX98_PTYS
-	if (device == MKDEV(TTYAUX_MAJOR,2)) {
-		int idr_ret;
-
-		/* find a device that is not in use. */
-		down(&allocated_ptys_lock);
-		if (!idr_pre_get(&allocated_ptys, GFP_KERNEL)) {
-			up(&allocated_ptys_lock);
-			return -ENOMEM;
-		}
-		idr_ret = idr_get_new(&allocated_ptys, NULL, &index);
-		if (idr_ret < 0) {
-			up(&allocated_ptys_lock);
-			if (idr_ret == -EAGAIN)
-				return -ENOMEM;
-			return -EIO;
-		}
-		if (index >= pty_limit) {
-			idr_remove(&allocated_ptys, index);
-			up(&allocated_ptys_lock);
-			return -EIO;
-		}
-		up(&allocated_ptys_lock);
-
-		driver = ptm_driver;
-		retval = init_dev(driver, index, &tty);
-		if (retval) {
-			down(&allocated_ptys_lock);
-			idr_remove(&allocated_ptys, index);
-			up(&allocated_ptys_lock);
-			return retval;
-		}
-
-		set_bit(TTY_PTY_LOCK, &tty->flags); /* LOCK THE SLAVE */
-		if (devpts_pty_new(tty->link))
-			retval = -ENOMEM;
-	} else
-#endif
-	{
 	driver = get_tty_driver(device, &index);
 	if (!driver)
 		return -ENODEV;
@@ -1404,7 +1379,6 @@ static int tty_open(struct inode * inode, struct file * filp)
 	retval = init_dev(driver, index, &tty);
 	if (retval)
 		return retval;
-	}
 
 	filp->private_data = tty;
 	file_move(filp, &tty->tty_files);
@@ -1431,15 +1405,6 @@ static int tty_open(struct inode * inode, struct file * filp)
 		printk(KERN_DEBUG "error %d in opening %s...", retval,
 		       tty->name);
 #endif
-
-#ifdef CONFIG_UNIX98_PTYS
-		if (index != -1) {
-			down(&allocated_ptys_lock);
-			idr_remove(&allocated_ptys, index);
-			up(&allocated_ptys_lock);
-		}
-#endif
-
 		release_dev(filp);
 		if (retval != -ERESTARTSYS)
 			return retval;
@@ -1467,6 +1432,62 @@ static int tty_open(struct inode * inode, struct file * filp)
 	return 0;
 }
 
+#ifdef CONFIG_UNIX98_PTYS
+static int ptmx_open(struct inode * inode, struct file * filp)
+{
+	struct tty_struct *tty;
+	int retval;
+	int index;
+	int idr_ret;
+
+	nonseekable_open(inode, filp);
+
+	/* find a device that is not in use. */
+	down(&allocated_ptys_lock);
+	if (!idr_pre_get(&allocated_ptys, GFP_KERNEL)) {
+		up(&allocated_ptys_lock);
+		return -ENOMEM;
+	}
+	idr_ret = idr_get_new(&allocated_ptys, NULL, &index);
+	if (idr_ret < 0) {
+		up(&allocated_ptys_lock);
+		if (idr_ret == -EAGAIN)
+			return -ENOMEM;
+		return -EIO;
+	}
+	if (index >= pty_limit) {
+		idr_remove(&allocated_ptys, index);
+		up(&allocated_ptys_lock);
+		return -EIO;
+	}
+	up(&allocated_ptys_lock);
+
+	retval = init_dev(ptm_driver, index, &tty);
+	if (retval)
+		goto out;
+
+	set_bit(TTY_PTY_LOCK, &tty->flags); /* LOCK THE SLAVE */
+	filp->private_data = tty;
+	file_move(filp, &tty->tty_files);
+
+	retval = -ENOMEM;
+	if (devpts_pty_new(tty->link))
+		goto out1;
+
+	check_tty_count(tty, "tty_open");
+	retval = ptm_driver->open(tty, filp);
+	if (!retval)
+		return 0;
+out1:
+	release_dev(filp);
+out:
+	down(&allocated_ptys_lock);
+	idr_remove(&allocated_ptys, index);
+	up(&allocated_ptys_lock);
+	return retval;
+}
+#endif
+
 static int tty_release(struct inode * inode, struct file * filp)
 {
 	lock_kernel();
@@ -2441,7 +2462,7 @@ static int __init tty_init(void)
 	class_simple_device_add(tty_class, MKDEV(TTYAUX_MAJOR, 1), NULL, "console");
 
 #ifdef CONFIG_UNIX98_PTYS
-	cdev_init(&ptmx_cdev, &tty_fops);
+	cdev_init(&ptmx_cdev, &ptmx_fops);
 	if (cdev_add(&ptmx_cdev, MKDEV(TTYAUX_MAJOR, 2), 1) ||
 	    register_chrdev_region(MKDEV(TTYAUX_MAJOR, 2), 1, "/dev/ptmx") < 0)
 		panic("Couldn't register /dev/ptmx driver\n");