RFC: mcu_tty: Trying to open /dev/ttyUSB0 and lock access from a kernel driver

[Ulf Samuelsson]

Trying to open /dev/ttyUSB from a kernel driver (which works), but 
locking the
serial port so noone else can access it does not work.
Any advice would be appreciated.

BACKGROUND:

I have a piece of hardware where I/O functions are implemented in a 
slave flash microcontroller (A)
which controls various functions (LEDs, buttons etc.) using a multiplex 
protocol on a serial port.
It also bridges data to a second slave microcontroller (B) by forwarding 
any data to a second serial port
on the slave microcontroller.

The driver is opening a serial port (currently /dev/ttyUSB0), and 
exports several serial port interfaces (/dev/ttyP[0-3])
where each serial port controls a single function on the slave 
microcontroller.
Other drivers or applications will communicate with the slave 
microcontroller by opening a /dev/ttyP# "serial port".
The serial port can be selected through a module parameter.

Whenever the driver  sends data to its "serial port", the multiplexing 
driver will encapsulate
the data in a packet, before sending it out on the H/W serial port.

The simple packet structure looks like:

struct packet {
    char    address;
    char    length;        /* including packet header */
    char    data[0..length-1-PACKET_HEADER_LENGTH];
    char    crc;
}

Whenever a packet is sent to a target, it will respond with a target 
specific acknowledge byte.

There is a significant S/W package which expects a standard serial port 
to microcontroller (B).
This S/W package does not expect to add the packet protocol,so to avoid 
a rewrite,
the multiplexing driver should do the encapsulation, and handle the ack.

There are a total of 4 targets addressed this way.

The four serial ports /dev/ttyP[0-3] which can be opened by other 
drivers are used as follows:

/dev/ttyP0:    LED driver which looks like a standard LED driver 
creating /sys/class/leds entries.
/dev/ttyP1:    Button handler reporting to the input subsystem
/dev/ttyP2:    Will not go into details
/dev/ttyP3:    Used for communicating with microcontroller B.

So far, I have a driver which will open /dev/ttyUSB (using filp_open) 
and send an initialization message.
It will then register four serial ports /dev/ttyP[0-3].

I can send data to the serial ports
$ echo XX > /dev/ttyP0
and the data will be sent to /dev/ttyUSB0 using vfs_write in the driver.

Can also open /dev/ttyP0 in minicom and can communicate.

I would like to lock the serial port excluively, so noone else can open it.

Here is my initialization.
If I first load the module which opens /dev/ttyUSB, I can still then 
open minicom and communicate over the port

==========================================================
How can I block minicom (and other applications/drivers) from accessing 
/dev/ttyUSB0?
I tried extracting the flock code from the kernel, and copied stuff 
which was not exported to the driver.
It seems to run properly, but locking does not work as expected.

Any clues why it will not work?
Other comments?


/* ================= INIT/EXIT ROUTINES ======================= */
#if    DEBUG
#define    PRINTK_DBG(...)    printk(__VA_ARGS__)
#else
#define    PRINTK_DBG(x)
#endif

/* copied from kernel, since it is not exported */
static inline int flock_translate_cmd(int cmd) {
    if (cmd & LOCK_MAND)
        return cmd & (LOCK_MAND | LOCK_RW);
    switch (cmd) {
    case LOCK_SH:
        return F_RDLCK;
    case LOCK_EX:
        return F_WRLCK;
    case LOCK_UN:
        return F_UNLCK;
    }
    return -EINVAL;
}

/* copied from kernel, since it is not exported */
static struct file_lock *
flock_make_lock(struct file *filp, unsigned int cmd)
{
    struct file_lock *fl;
    int type = flock_translate_cmd(cmd);

    if (type < 0)
        return ERR_PTR(type);

    fl = locks_alloc_lock();
    if (fl == NULL)
        return ERR_PTR(-ENOMEM);

    fl->fl_file = filp;
    fl->fl_owner = filp;
    fl->fl_pid = current->tgid;
    fl->fl_flags = FL_FLOCK;
    fl->fl_type = type;
    fl->fl_end = OFFSET_MAX;

    return fl;
}

void closetty (void)
{
    if (realtty) {
        put_unused_fd(common->fd);
        common->fd = -1;
        /* A lock should be removed at close */
        if (realtty != NULL)
            filp_close(realtty, 0);
        realtty = NULL;
        common->ftty = NULL;
    }
}

static int opentty (void)
{
    loff_t pos = 0;
    int    retval;
    unsigned flags;
    int    fd;
    struct file_lock *lock;
    mm_segment_t old_fs;

    old_fs = get_fs();
    set_fs(get_ds());


    /* Get a new file descriptor */
    flags = O_RDWR | O_NOCTTY | O_NDELAY | O_SYNC | O_APPEND | O_NONBLOCK;
    PRINTK_DBG(KERN_INFO "%s: Allocating a file descriptor\n", __MODULE__);
    fd = get_unused_fd_flags(flags);
    if (fd < 0) {
        printk(KERN_ERR "%s: could not allocate file descriptor for 
'%s'\n", __MODULE__ ,device);
        retval = -ENODEV;
        goto err;
    }

    /* Open the shared serial port */
    PRINTK_DBG(KERN_INFO "%s: Opening %s\n", __MODULE__, device);
    common->name = device;
    realtty = filp_open(device, flags, 0644);
    if (IS_ERR(realtty)) {
        printk(KERN_ERR "%s: cannot open '%s'\n", __MODULE__, device);
        retval = PTR_ERR(realtty);
        realtty = NULL;
        closetty();
        goto    err;
    }

    PRINTK_DBG(KERN_INFO "%s: '%s' successfully opened\n", __MODULE__, 
device);

    /* Notification*/
    PRINTK_DBG(KERN_INFO "%s: fsnotify on %s\n", __MODULE__, device);
    fsnotify_open(realtty);

    /* Connect file descriptor with file */
    PRINTK_DBG(KERN_INFO "%s: Connect %s to file descriptor\n", 
__MODULE__, device);
    fd_install(fd, realtty);
    common->fd = fd;

    /* Create a lock for the file */
    PRINTK_DBG(KERN_INFO "%s: Create Lock for %s\n", __MODULE__, device);
    lock = flock_make_lock(realtty, LOCK_EX);
    if (IS_ERR(lock)) {
        PRINTK_DBG(KERN_INFO "%s: Create Lock for %s failed\n", 
__MODULE__, device);
        retval = PTR_ERR(lock);
        closetty();
        goto    err;
    }

    /* Exlusive lock on the file, remove lock on close */
    lock->fl_flags            |= (FL_POSIX | FL_CLOSE);    /* Remove 
lock when file is closed */
    PRINTK_DBG(KERN_INFO "%s: Lock %s\n", __MODULE__, device);
    retval = vfs_lock_file(realtty, F_SETLK, lock, NULL);
    if (retval) {
        if (retval == EACCES || retval == EAGAIN) {
            printk(KERN_ERR "%s: %s already locked by another 
process\n", __MODULE__, device);
        } else {
            /* Handle unexpected error (Not Yet) */;
        }
        /* We need to close the device */
        closetty();
        goto    err;
    } else {
        common->fl = lock;
    }

    /* Set speed to 115200 BAUD */
    /* TODO: Implement speed change */
    PRINTK_DBG(KERN_INFO "%s: Setting %s to %d BAUD\n", __MODULE__, 
device, 115200);
    tty_setspeed(realtty, 115200);
    /* Should check results */

#if    defined(DEBUG)
    /* This write is only for debugging the driver */
    vfs_write(realtty, "SUCCESS\n", 8, &pos);
    /* Should check results ? */
#endif
    retval    = 0;

err:
    set_fs(old_fs); //Reset to save FS
    return    retval;
}

static int __init mcu_init(void)
{
    int i;
    struct mcu_common    *loc_common;
    struct tty_info    *info;
    struct tty_driver *driver;
    int retval;
    /********************************************************/
    /* this is the first time this port is opened        */
    /* do any hardware initialization needed here        */
    /********************************************************/

    printk(KERN_INFO DRIVER_DESC " " DRIVER_VERSION);
    loc_common = kzalloc(sizeof(*loc_common), GFP_KERNEL);
    if (!loc_common) {
        retval = -ENOMEM;
        goto    nomem;
    }

    common = loc_common;

    if (opentty() < 0) {
        retval = -ENODEV;
        goto    err;
    }
    /*********************************************************/

    PRINTK_DBG(KERN_INFO "%s: Allocate tty driver\n", __MODULE__);
    /* allocate the tty driver */
    /* TODO: Do we want to have a module parameter for number of 
devices ? */
    driver = alloc_tty_driver(MCU_TTY_MINORS);
    if (!driver) {
        retval = -ENOMEM;
        goto    err_close;
    }

    PRINTK_DBG(KERN_INFO "MCU Shared Serial Port Driver Init\n");

    /* initialize the tty driver */
    driver->owner            = THIS_MODULE;
    driver->driver_name        = __MODULE__;
    driver->name            = "ttyP";
    driver->major            = MCU_TTY_MAJOR,
    driver->minor_start        = 64;
    driver->type            = TTY_DRIVER_TYPE_SERIAL,
    driver->subtype            = SERIAL_TYPE_NORMAL,
    driver->init_termios        = tty_std_termios;
    driver->init_termios.c_cflag    = B115200 | CS8 | CREAD | HUPCL | 
CLOCAL;
    driver->flags            =
            TTY_DRIVER_RESET_TERMIOS |
            TTY_DRIVER_REAL_RAW |
            TTY_DRIVER_DYNAMIC_DEV,
    driver->init_termios.c_ispeed    = 115200;
    driver->init_termios.c_ospeed    = 115200;

    PRINTK_DBG(KERN_INFO "%s: Set Operations\n", __MODULE__);

    tty_set_operations(driver, &serial_ops);
    mcu_tty_driver = driver;

    /*********************************************************/
    /* first time accessing this device, let's create it */
    PRINTK_DBG(KERN_INFO "%s: Create the device\n", __MODULE__);

    PRINTK_DBG(KERN_INFO "%s: Initialize Ports\n", __MODULE__);

    for (i = 0, info = &loc_common->tty_infos[0] ; i < MCU_TTY_MINORS ; 
++i, info++) {
        PRINTK_DBG(KERN_INFO "/dev/%s%d is multiplexed on 
%s\n",driver->name, i, device);
        info->enabled    = 1;
        info->line    = i;
        info->name    = driver->name;
        info->opencount    = 0;
        info->board    = loc_common;
        sema_init(&info->sem,     1);
        tty_port_init(&info->port);
        /* TODO: Check result */
        tty_port_register_device(&info->port, driver, i, NULL);
        /* TODO: Check result */
    }
    PRINTK_DBG(KERN_INFO "%s: Register the tty driver\n", __MODULE__);

    /* register the tty driver */
    retval = tty_register_driver(driver);
    if (retval) {
        printk(KERN_ERR "failed to register %s driver\n", 
driver->driver_name);
        goto    no_register_driver;
    }
    PRINTK_DBG(KERN_INFO DRIVER_DESC " " "SUCCESS\n");
    return    retval;

no_register_driver:
    put_tty_driver(mcu_tty_driver);

// no_register_device:
    for (i = 0; i < MCU_TTY_MINORS; ++i) {
        PRINTK_DBG(KERN_INFO "%s: unregistering device %d\n", 
__MODULE__, i);
        tty_unregister_device(mcu_tty_driver, i);
        tty_port_destroy(&common->tty_infos[i].port);
    }
err_close:
    closetty();
err:
    kfree(common);
    common = NULL;
nomem:
    return retval;
}

static void __exit mcu_exit(void)
{
    struct tty_info *info;
    int i;
    PRINTK_DBG(KERN_INFO "%s: unregistering devices\n", __MODULE__);
    for (i = 0; i < MCU_TTY_MINORS; ++i) {
        PRINTK_DBG(KERN_INFO "%s: unregistering device %d\n", 
__MODULE__, i);
        tty_unregister_device(mcu_tty_driver, i);
        tty_port_destroy(&common->tty_infos[i].port);
    }
    PRINTK_DBG(KERN_INFO "Unregistering devices - complete\n");
    if (mcu_tty_driver == NULL) {
        PRINTK_DBG(KERN_INFO "%s: unregistering driver - NULL\n", 
__MODULE__);
    }
    PRINTK_DBG(KERN_INFO "Unregistering driver\n");
    tty_unregister_driver(mcu_tty_driver);
    put_tty_driver(mcu_tty_driver);

    PRINTK_DBG(KERN_INFO "Freeing up Memory\n");
    /* shut down and free the memory */
    for (i = 0, info = &common->tty_infos[0]; i < MCU_TTY_MINORS; ++i, 
info++) {
        if (info) {
            /* close the port */
            while (info->opencount)
                do_close(info);
            kfree(info);
        }
    }

    /* The lock should be released automatically */
    /* Release file descriptor and close ${device} file */
    closetty();
    kfree(common);
    common = NULL;
}

module_init(mcu_init);
module_exit(mcu_exit);

/* ===================================================== */

Best regards
Ulf Samuelsson