227 lines
6.9 KiB
C
227 lines
6.9 KiB
C
// SPDX-License-Identifier: MIT
|
|
/*
|
|
* Copyright © 2024 Intel Corporation
|
|
*/
|
|
|
|
#include <drm/ttm/ttm_backup.h>
|
|
|
|
#include <linux/export.h>
|
|
#include <linux/swap.h>
|
|
|
|
#include "ttm_pool_internal.h"
|
|
|
|
/*
|
|
* Need to map shmem indices to handle since a handle value
|
|
* of 0 means error, following the swp_entry_t convention.
|
|
*/
|
|
static unsigned long ttm_backup_shmem_idx_to_handle(pgoff_t idx)
|
|
{
|
|
return (unsigned long)idx + 1;
|
|
}
|
|
|
|
static pgoff_t ttm_backup_handle_to_shmem_idx(pgoff_t handle)
|
|
{
|
|
return handle - 1;
|
|
}
|
|
|
|
/**
|
|
* ttm_backup_drop() - release memory associated with a handle
|
|
* @backup: The struct backup pointer used to obtain the handle
|
|
* @handle: The handle obtained from the @backup_page function.
|
|
*/
|
|
void ttm_backup_drop(struct file *backup, pgoff_t handle)
|
|
{
|
|
loff_t start = ttm_backup_handle_to_shmem_idx(handle);
|
|
|
|
start <<= PAGE_SHIFT;
|
|
shmem_truncate_range(file_inode(backup), start,
|
|
start + PAGE_SIZE - 1);
|
|
}
|
|
|
|
/**
|
|
* ttm_backup_copy_page() - Copy the contents of a previously backed
|
|
* up page
|
|
* @backup: The struct backup pointer used to back up the page.
|
|
* @dst: The struct page to copy into.
|
|
* @handle: The handle returned when the page was backed up.
|
|
* @intr: Try to perform waits interruptible or at least killable.
|
|
* @additional_gfp: GFP mask to add to the default GFP mask if any.
|
|
*
|
|
* Return: 0 on success, Negative error code on failure, notably
|
|
* -EINTR if @intr was set to true and a signal is pending.
|
|
*/
|
|
int ttm_backup_copy_page(struct file *backup, struct page *dst,
|
|
pgoff_t handle, bool intr, gfp_t additional_gfp)
|
|
{
|
|
struct address_space *mapping = backup->f_mapping;
|
|
struct folio *from_folio;
|
|
pgoff_t idx = ttm_backup_handle_to_shmem_idx(handle);
|
|
|
|
from_folio = shmem_read_folio_gfp(mapping, idx, mapping_gfp_mask(mapping)
|
|
| additional_gfp);
|
|
if (IS_ERR(from_folio))
|
|
return PTR_ERR(from_folio);
|
|
|
|
copy_highpage(dst, folio_file_page(from_folio, idx));
|
|
folio_put(from_folio);
|
|
|
|
return 0;
|
|
}
|
|
|
|
/**
|
|
* ttm_backup_backup_folio() - Backup a folio
|
|
* @backup: The struct backup pointer to use.
|
|
* @folio: The folio to back up.
|
|
* @order: The allocation order of @folio. Since TTM allocates higher-order
|
|
* pages without __GFP_COMP, folio_nr_pages(@folio) would always
|
|
* return 1; the caller must pass the true order explicitly.
|
|
* @writeback: Whether to perform immediate writeback of the folio's pages.
|
|
* This may have performance implications.
|
|
* @idx: A unique integer for the first page of the folio and each struct backup.
|
|
* This allows the backup implementation to avoid managing
|
|
* its address space separately.
|
|
* @folio_gfp: The gfp value used when the folio was allocated.
|
|
* Currently unused.
|
|
* @alloc_gfp: The gfp to be used when allocating memory.
|
|
* @nr_pages_backed: Output. On a successful return, set to the number of
|
|
* pages actually backed up, which may be less than (1 << @order)
|
|
* if an -ENOMEM was encountered mid-folio.
|
|
*
|
|
* Context: If called from reclaim context, the caller needs to
|
|
* assert that the shrinker gfp has __GFP_FS set, to avoid
|
|
* deadlocking on lock_page(). If @writeback is set to true and
|
|
* called from reclaim context, the caller also needs to assert
|
|
* that the shrinker gfp has __GFP_IO set, since without it,
|
|
* we're not allowed to start backup IO.
|
|
*
|
|
* Return: A handle for the first backed-up page on success (handles for
|
|
* subsequent pages follow sequentially). -ENOMEM if no pages could be backed
|
|
* up. Any other negative error code if a non-ENOMEM failure occurred; in that
|
|
* case any pages backed up so far are truncated before returning.
|
|
*/
|
|
s64
|
|
ttm_backup_backup_folio(struct file *backup, struct folio *folio,
|
|
unsigned int order, bool writeback, pgoff_t idx,
|
|
gfp_t folio_gfp, gfp_t alloc_gfp,
|
|
pgoff_t *nr_pages_backed)
|
|
{
|
|
struct address_space *mapping = backup->f_mapping;
|
|
int nr_pages = 1 << order;
|
|
struct folio *to_folio;
|
|
int ret, i;
|
|
|
|
*nr_pages_backed = 0;
|
|
|
|
for (i = 0; i < nr_pages; ) {
|
|
int to_nr, j;
|
|
|
|
/*
|
|
* Only inject past the first subpage so *nr_pages_backed is
|
|
* always > 0 here, matching a genuine mid-compound -ENOMEM
|
|
* and driving the caller's reactive split fallback instead
|
|
* of an early, no-progress failure.
|
|
*/
|
|
if (IS_ENABLED(CONFIG_FAULT_INJECTION) && i &&
|
|
ttm_backup_fault_inject_folio())
|
|
to_folio = ERR_PTR(-ENOMEM);
|
|
else
|
|
to_folio = shmem_read_folio_gfp(mapping, idx + i, alloc_gfp);
|
|
if (IS_ERR(to_folio)) {
|
|
int err = PTR_ERR(to_folio);
|
|
|
|
if (err == -ENOMEM && *nr_pages_backed)
|
|
return ttm_backup_shmem_idx_to_handle(idx);
|
|
|
|
if (*nr_pages_backed) {
|
|
shmem_truncate_range(file_inode(backup),
|
|
(loff_t)idx << PAGE_SHIFT,
|
|
((loff_t)(idx + i) << PAGE_SHIFT) - 1);
|
|
/*
|
|
* The pages just truncated are no longer
|
|
* backed up; don't let the caller mistake
|
|
* them for valid handles.
|
|
*/
|
|
*nr_pages_backed = 0;
|
|
}
|
|
return err;
|
|
}
|
|
|
|
to_nr = min_t(int, nr_pages - i,
|
|
folio_next_index(to_folio) - (idx + i));
|
|
|
|
folio_mark_accessed(to_folio);
|
|
folio_lock(to_folio);
|
|
folio_mark_dirty(to_folio);
|
|
|
|
for (j = 0; j < to_nr; j++)
|
|
copy_highpage(folio_file_page(to_folio, idx + i + j),
|
|
folio_page(folio, i + j));
|
|
|
|
if (writeback && !folio_mapped(to_folio) &&
|
|
folio_clear_dirty_for_io(to_folio)) {
|
|
folio_set_reclaim(to_folio);
|
|
ret = shmem_writeout(to_folio, NULL, NULL);
|
|
if (!folio_test_writeback(to_folio))
|
|
folio_clear_reclaim(to_folio);
|
|
if (ret == AOP_WRITEPAGE_ACTIVATE)
|
|
folio_unlock(to_folio);
|
|
} else {
|
|
folio_unlock(to_folio);
|
|
}
|
|
|
|
folio_put(to_folio);
|
|
i += to_nr;
|
|
*nr_pages_backed = i;
|
|
}
|
|
|
|
return ttm_backup_shmem_idx_to_handle(idx);
|
|
}
|
|
|
|
/**
|
|
* ttm_backup_fini() - Free the struct backup resources after last use.
|
|
* @backup: Pointer to the struct backup whose resources to free.
|
|
*
|
|
* After a call to this function, it's illegal to use the @backup pointer.
|
|
*/
|
|
void ttm_backup_fini(struct file *backup)
|
|
{
|
|
fput(backup);
|
|
}
|
|
|
|
/**
|
|
* ttm_backup_bytes_avail() - Report the approximate number of bytes of backup space
|
|
* left for backup.
|
|
*
|
|
* This function is intended also for driver use to indicate whether a
|
|
* backup attempt is meaningful.
|
|
*
|
|
* Return: An approximate size of backup space available.
|
|
*/
|
|
u64 ttm_backup_bytes_avail(void)
|
|
{
|
|
/*
|
|
* The idea behind backing up to shmem is that shmem objects may
|
|
* eventually be swapped out. So no point swapping out if there
|
|
* is no or low swap-space available. But the accuracy of this
|
|
* number also depends on shmem actually swapping out backed-up
|
|
* shmem objects without too much buffering.
|
|
*/
|
|
return (u64)get_nr_swap_pages() << PAGE_SHIFT;
|
|
}
|
|
EXPORT_SYMBOL_GPL(ttm_backup_bytes_avail);
|
|
|
|
/**
|
|
* ttm_backup_shmem_create() - Create a shmem-based struct backup.
|
|
* @size: The maximum size (in bytes) to back up.
|
|
*
|
|
* Create a backup utilizing shmem objects.
|
|
*
|
|
* Return: A pointer to a struct file on success,
|
|
* an error pointer on error.
|
|
*/
|
|
struct file *ttm_backup_shmem_create(loff_t size)
|
|
{
|
|
return shmem_file_setup("ttm shmem backup", size,
|
|
EMPTY_VMA_FLAGS);
|
|
}
|