[PATCH] linux_usbfs: Treat EINVAL as ENOTTY in cases where we may get ENOTTY

[Hans de Goede]

Older 64 bit kernels will return EINVAL to 32 bits userspace for unknown
ioctls. So in places where we check for ENOTTY also check for EINVAL, and
use the fallback path for EINVAL too.

Note that if there really is an invalid parameter in the
detach_kernel_driver_and_claim case, then the fallback path will detect and
report this.

Signed-off-by: Hans de Goede <***@redhat.com>
---
libusb/os/linux_usbfs.c | 13 ++++++-------
1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/libusb/os/linux_usbfs.c b/libusb/os/linux_usbfs.c
index 8962aba..00c8090 100644
--- a/libusb/os/linux_usbfs.c
+++ b/libusb/os/linux_usbfs.c
@@ -1285,7 +1285,8 @@ static int op_open(struct libusb_device_handle *handle)

r = ioctl(hpriv->fd, IOCTL_USBFS_GET_CAPABILITIES, &hpriv->caps);
if (r < 0) {
- if (errno == ENOTTY)
+ /* Older 64 bit kernels return EINVAL to 32 bit apps */
+ if (errno == ENOTTY || errno == EINVAL)
usbi_dbg("getcap not available");
else
usbi_err(HANDLE_CTX(handle), "getcap failed (%d)", errno);
@@ -1631,15 +1632,13 @@ static int detach_kernel_driver_and_claim(struct libusb_device_handle *handle,
strcpy(dc.driver, "usbfs");
dc.flags = USBFS_DISCONNECT_CLAIM_EXCEPT_DRIVER;
r = ioctl(fd, IOCTL_USBFS_DISCONNECT_CLAIM, &dc);
- if (r == 0 || (r != 0 && errno != ENOTTY)) {
- if (r == 0)
- return 0;
-
+ if (r == 0)
+ return 0;
+ /* Older 64 bit kernels return EINVAL to 32 bit apps */
+ if (errno != ENOTTY && errno != EINVAL) {
switch (errno) {
case EBUSY:
return LIBUSB_ERROR_BUSY;
- case EINVAL:
- return LIBUSB_ERROR_INVALID_PARAM;
case ENODEV:
return LIBUSB_ERROR_NO_DEVICE;
}

[Hans de Goede]

Older 64 bit kernels will return EINVAL to 32 bits userspace for unknown
ioctls. So in places where we check for ENOTTY also check for EINVAL, and
use the fallback path for EINVAL too.

Note that if there really is an invalid parameter in the
detach_kernel_driver_and_claim case, then the fallback path will detect and
report this.

Signed-off-by: Hans de Goede <***@redhat.com>
---
libusb/os/linux_usbfs.c | 13 ++++++-------
1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/libusb/os/linux_usbfs.c b/libusb/os/linux_usbfs.c
index d654741..2110539 100644
--- a/libusb/os/linux_usbfs.c
+++ b/libusb/os/linux_usbfs.c
@@ -1285,7 +1285,8 @@ static int op_open(struct libusb_device_handle *handle)

r = ioctl(hpriv->fd, IOCTL_USBFS_GET_CAPABILITIES, &hpriv->caps);
if (r < 0) {
- if (errno == ENOTTY)
+ /* Older 64 bit kernels return EINVAL to 32 bit apps */
+ if (errno == ENOTTY || errno == EINVAL)
usbi_dbg("getcap not available");
else
usbi_err(HANDLE_CTX(handle), "getcap failed (%d)", errno);
@@ -1636,15 +1637,13 @@ static int detach_kernel_driver_and_claim(struct libusb_device_handle *handle,
strcpy(dc.driver, "usbfs");
dc.flags = USBFS_DISCONNECT_CLAIM_EXCEPT_DRIVER;
r = ioctl(fd, IOCTL_USBFS_DISCONNECT_CLAIM, &dc);
- if (r == 0 || (r != 0 && errno != ENOTTY)) {
- if (r == 0)
- return 0;
-
+ if (r == 0)
+ return 0;
+ /* Older 64 bit kernels return EINVAL to 32 bit apps */
+ if (errno != ENOTTY && errno != EINVAL) {
switch (errno) {
case EBUSY:
return LIBUSB_ERROR_BUSY;
- case EINVAL:
- return LIBUSB_ERROR_INVALID_PARAM;
case ENODEV:
return LIBUSB_ERROR_NO_DEVICE;
}