301 lines
8.2 KiB
C
301 lines
8.2 KiB
C
// SPDX-License-Identifier: GPL-2.0
|
|
/*
|
|
* I/O and data path helper functionality.
|
|
*
|
|
* Borrowed from NFS Copyright (c) 2016 Trond Myklebust
|
|
*/
|
|
|
|
#include <linux/kernel.h>
|
|
#include <linux/netfs.h>
|
|
#include "internal.h"
|
|
|
|
struct netfs_wb_waiter {
|
|
struct list_head link; /* Link in ictx->wb_queue */
|
|
struct task_struct *waiter; /* Waiter task; cleared when lock granted */
|
|
};
|
|
|
|
/*
|
|
* inode_dio_wait_interruptible - wait for outstanding DIO requests to finish
|
|
* @inode: inode to wait for
|
|
*
|
|
* Waits for all pending direct I/O requests to finish so that we can
|
|
* proceed with a truncate or equivalent operation.
|
|
*
|
|
* Must be called under a lock that serializes taking new references
|
|
* to i_dio_count, usually by inode->i_mutex.
|
|
*/
|
|
static int netfs_inode_dio_wait_interruptible(struct inode *inode)
|
|
{
|
|
if (inode_dio_finished(inode))
|
|
return 0;
|
|
|
|
inode_dio_wait_interruptible(inode);
|
|
return !inode_dio_finished(inode) ? -ERESTARTSYS : 0;
|
|
}
|
|
|
|
/* Call with exclusively locked inode->i_rwsem */
|
|
static int netfs_block_o_direct(struct netfs_inode *ictx)
|
|
{
|
|
if (!test_bit(NETFS_ICTX_ODIRECT, &ictx->flags))
|
|
return 0;
|
|
clear_bit(NETFS_ICTX_ODIRECT, &ictx->flags);
|
|
return netfs_inode_dio_wait_interruptible(&ictx->inode);
|
|
}
|
|
|
|
/**
|
|
* netfs_start_io_read - declare the file is being used for buffered reads
|
|
* @inode: file inode
|
|
*
|
|
* Declare that a buffered read operation is about to start, and ensure
|
|
* that we block all direct I/O.
|
|
* On exit, the function ensures that the NETFS_ICTX_ODIRECT flag is unset,
|
|
* and holds a shared lock on inode->i_rwsem to ensure that the flag
|
|
* cannot be changed.
|
|
* In practice, this means that buffered read operations are allowed to
|
|
* execute in parallel, thanks to the shared lock, whereas direct I/O
|
|
* operations need to wait to grab an exclusive lock in order to set
|
|
* NETFS_ICTX_ODIRECT.
|
|
* Note that buffered writes and truncates both take a write lock on
|
|
* inode->i_rwsem, meaning that those are serialised w.r.t. the reads.
|
|
*/
|
|
int netfs_start_io_read(struct inode *inode)
|
|
__acquires(inode->i_rwsem)
|
|
{
|
|
struct netfs_inode *ictx = netfs_inode(inode);
|
|
|
|
/* Be an optimist! */
|
|
if (down_read_interruptible(&inode->i_rwsem) < 0)
|
|
return -ERESTARTSYS;
|
|
if (test_bit(NETFS_ICTX_ODIRECT, &ictx->flags) == 0)
|
|
return 0;
|
|
up_read(&inode->i_rwsem);
|
|
|
|
/* Slow path.... */
|
|
if (down_write_killable(&inode->i_rwsem) < 0)
|
|
return -ERESTARTSYS;
|
|
if (netfs_block_o_direct(ictx) < 0) {
|
|
up_write(&inode->i_rwsem);
|
|
return -ERESTARTSYS;
|
|
}
|
|
downgrade_write(&inode->i_rwsem);
|
|
return 0;
|
|
}
|
|
EXPORT_SYMBOL(netfs_start_io_read);
|
|
|
|
/**
|
|
* netfs_end_io_read - declare that the buffered read operation is done
|
|
* @inode: file inode
|
|
*
|
|
* Declare that a buffered read operation is done, and release the shared
|
|
* lock on inode->i_rwsem.
|
|
*/
|
|
void netfs_end_io_read(struct inode *inode)
|
|
__releases(inode->i_rwsem)
|
|
{
|
|
up_read(&inode->i_rwsem);
|
|
}
|
|
EXPORT_SYMBOL(netfs_end_io_read);
|
|
|
|
/**
|
|
* netfs_start_io_write - declare the file is being used for buffered writes
|
|
* @inode: file inode
|
|
*
|
|
* Declare that a buffered read operation is about to start, and ensure
|
|
* that we block all direct I/O.
|
|
*/
|
|
int netfs_start_io_write(struct inode *inode)
|
|
__acquires(inode->i_rwsem)
|
|
{
|
|
struct netfs_inode *ictx = netfs_inode(inode);
|
|
|
|
if (down_write_killable(&inode->i_rwsem) < 0)
|
|
return -ERESTARTSYS;
|
|
if (netfs_block_o_direct(ictx) < 0) {
|
|
up_write(&inode->i_rwsem);
|
|
return -ERESTARTSYS;
|
|
}
|
|
downgrade_write(&inode->i_rwsem);
|
|
return 0;
|
|
}
|
|
EXPORT_SYMBOL(netfs_start_io_write);
|
|
|
|
/**
|
|
* netfs_end_io_write - declare that the buffered write operation is done
|
|
* @inode: file inode
|
|
*
|
|
* Declare that a buffered write operation is done, and release the
|
|
* lock on inode->i_rwsem.
|
|
*/
|
|
void netfs_end_io_write(struct inode *inode)
|
|
__releases(inode->i_rwsem)
|
|
{
|
|
up_read(&inode->i_rwsem);
|
|
}
|
|
EXPORT_SYMBOL(netfs_end_io_write);
|
|
|
|
/* Call with exclusively locked inode->i_rwsem */
|
|
static int netfs_block_buffered(struct inode *inode)
|
|
{
|
|
struct netfs_inode *ictx = netfs_inode(inode);
|
|
int ret;
|
|
|
|
if (!test_bit(NETFS_ICTX_ODIRECT, &ictx->flags)) {
|
|
set_bit(NETFS_ICTX_ODIRECT, &ictx->flags);
|
|
if (inode->i_mapping->nrpages != 0) {
|
|
unmap_mapping_range(inode->i_mapping, 0, 0, 0);
|
|
ret = filemap_fdatawait(inode->i_mapping);
|
|
if (ret < 0) {
|
|
clear_bit(NETFS_ICTX_ODIRECT, &ictx->flags);
|
|
return ret;
|
|
}
|
|
}
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
/**
|
|
* netfs_start_io_direct - declare the file is being used for direct i/o
|
|
* @inode: file inode
|
|
*
|
|
* Declare that a direct I/O operation is about to start, and ensure
|
|
* that we block all buffered I/O.
|
|
* On exit, the function ensures that the NETFS_ICTX_ODIRECT flag is set,
|
|
* and holds a shared lock on inode->i_rwsem to ensure that the flag
|
|
* cannot be changed.
|
|
* In practice, this means that direct I/O operations are allowed to
|
|
* execute in parallel, thanks to the shared lock, whereas buffered I/O
|
|
* operations need to wait to grab an exclusive lock in order to clear
|
|
* NETFS_ICTX_ODIRECT.
|
|
* Note that buffered writes and truncates both take a write lock on
|
|
* inode->i_rwsem, meaning that those are serialised w.r.t. O_DIRECT.
|
|
*/
|
|
int netfs_start_io_direct(struct inode *inode)
|
|
__acquires(inode->i_rwsem)
|
|
{
|
|
struct netfs_inode *ictx = netfs_inode(inode);
|
|
int ret;
|
|
|
|
/* Be an optimist! */
|
|
if (down_read_interruptible(&inode->i_rwsem) < 0)
|
|
return -ERESTARTSYS;
|
|
if (test_bit(NETFS_ICTX_ODIRECT, &ictx->flags) != 0)
|
|
return 0;
|
|
up_read(&inode->i_rwsem);
|
|
|
|
/* Slow path.... */
|
|
if (down_write_killable(&inode->i_rwsem) < 0)
|
|
return -ERESTARTSYS;
|
|
ret = netfs_block_buffered(inode);
|
|
if (ret < 0) {
|
|
up_write(&inode->i_rwsem);
|
|
return ret;
|
|
}
|
|
downgrade_write(&inode->i_rwsem);
|
|
return 0;
|
|
}
|
|
EXPORT_SYMBOL(netfs_start_io_direct);
|
|
|
|
/**
|
|
* netfs_end_io_direct - declare that the direct i/o operation is done
|
|
* @inode: file inode
|
|
*
|
|
* Declare that a direct I/O operation is done, and release the shared
|
|
* lock on inode->i_rwsem.
|
|
*/
|
|
void netfs_end_io_direct(struct inode *inode)
|
|
__releases(inode->i_rwsem)
|
|
{
|
|
up_read(&inode->i_rwsem);
|
|
}
|
|
EXPORT_SYMBOL(netfs_end_io_direct);
|
|
|
|
/*
|
|
* Wait to have exclusive access to writeback.
|
|
*/
|
|
static bool netfs_wb_begin_wait(struct netfs_inode *ictx)
|
|
{
|
|
struct netfs_wb_waiter waiter = {};
|
|
struct task_struct *tsk = current;
|
|
bool got = false;
|
|
|
|
spin_lock(&ictx->lock);
|
|
|
|
if (test_and_set_bit_lock(NETFS_ICTX_WB_LOCK, &ictx->flags)) {
|
|
get_task_struct(tsk);
|
|
waiter.waiter = tsk;
|
|
list_add_tail(&waiter.link, &ictx->wb_queue);
|
|
} else {
|
|
got = true;
|
|
}
|
|
spin_unlock(&ictx->lock);
|
|
|
|
if (!got) {
|
|
for (;;) {
|
|
set_current_state(TASK_UNINTERRUPTIBLE);
|
|
/* Read waiter before accessing inode state. */
|
|
if (smp_load_acquire(&waiter.waiter) == NULL)
|
|
break;
|
|
schedule();
|
|
}
|
|
}
|
|
__set_current_state(TASK_RUNNING);
|
|
return true;
|
|
}
|
|
|
|
/**
|
|
* netfs_wb_begin - Begin writeback, waiting if need be
|
|
* @ictx: The inode to get writeback access on
|
|
* @nowait: Return failure immediately rather than waiting if true
|
|
*
|
|
* Begin writeback to an inode, waiting for exclusive access if @nowait is
|
|
* false. This prevents collection from being done out of order with respect
|
|
* to the issuance of write subrequests.
|
|
*
|
|
* Note that writeback may be ended in a different process (e.g. the collection
|
|
* function on a workqueue) than started it.
|
|
*
|
|
* Return: True if can proceed, false if denied.
|
|
*/
|
|
bool netfs_wb_begin(struct netfs_inode *ictx, bool nowait)
|
|
{
|
|
if (!test_and_set_bit_lock(NETFS_ICTX_WB_LOCK, &ictx->flags))
|
|
return true;
|
|
if (nowait) {
|
|
netfs_stat(&netfs_n_wb_lock_skip);
|
|
return false;
|
|
}
|
|
netfs_stat(&netfs_n_wb_lock_wait);
|
|
return netfs_wb_begin_wait(ictx);
|
|
}
|
|
EXPORT_SYMBOL(netfs_wb_begin);
|
|
|
|
/* netfs_wb_end - End writeback
|
|
* @ictx: The inode we have writeback access to
|
|
*
|
|
* End writeback access on an inode, waking up the next writeback request.
|
|
*/
|
|
void netfs_wb_end(struct netfs_inode *ictx)
|
|
{
|
|
struct netfs_wb_waiter *waiter;
|
|
struct task_struct *tsk;
|
|
|
|
WARN_ON_ONCE(!test_bit(NETFS_ICTX_WB_LOCK, &ictx->flags));
|
|
|
|
spin_lock(&ictx->lock);
|
|
|
|
waiter = list_first_entry_or_null(&ictx->wb_queue, struct netfs_wb_waiter, link);
|
|
if (waiter) {
|
|
list_del(&waiter->link);
|
|
tsk = waiter->waiter;
|
|
/* Write inode state before clearing waiter. */
|
|
smp_store_release(&waiter->waiter, NULL);
|
|
wake_up_process(tsk);
|
|
put_task_struct(tsk);
|
|
} else {
|
|
clear_bit_unlock(NETFS_ICTX_WB_LOCK, &ictx->flags);
|
|
}
|
|
|
|
spin_unlock(&ictx->lock);
|
|
}
|
|
EXPORT_SYMBOL(netfs_wb_end);
|