739 lines
21 KiB
C
Executable File
739 lines
21 KiB
C
Executable File
/*
|
|
* Copyright (c) 2006-2023, RT-Thread Development Team
|
|
*
|
|
* SPDX-License-Identifier: Apache-2.0
|
|
*
|
|
* Change Logs:
|
|
* Date Author Notes
|
|
* 2023-08-19 Shell Support PRIVATE mapping and COW
|
|
*/
|
|
|
|
#define DBG_TAG "mm.anon"
|
|
#define DBG_LVL DBG_INFO
|
|
#include <rtdbg.h>
|
|
|
|
#include <string.h>
|
|
#include "mm_private.h"
|
|
#include <mmu.h>
|
|
|
|
/**
|
|
* Anonymous Object directly represent the mappings without backup files in the
|
|
* aspace. Their only backup is in the aspace->pgtbl.
|
|
*/
|
|
|
|
typedef struct rt_private_ctx {
|
|
struct rt_mem_obj mem_obj;
|
|
rt_aspace_t backup_aspace;
|
|
/* both varea and aspace can holds a reference */
|
|
rt_atomic_t reference;
|
|
/* readonly `private` is shared object */
|
|
long readonly;
|
|
} *rt_private_ctx_t;
|
|
|
|
rt_inline rt_aspace_t _anon_obj_get_backup(rt_mem_obj_t mobj)
|
|
{
|
|
rt_private_ctx_t pctx;
|
|
rt_aspace_t backup;
|
|
pctx = rt_container_of(mobj, struct rt_private_ctx, mem_obj);
|
|
backup = pctx->backup_aspace;
|
|
return backup;
|
|
}
|
|
|
|
rt_inline rt_atomic_t *_anon_obj_get_reference(rt_mem_obj_t mobj)
|
|
{
|
|
rt_private_ctx_t pctx;
|
|
pctx = rt_container_of(mobj, struct rt_private_ctx, mem_obj);
|
|
return &pctx->reference;
|
|
}
|
|
|
|
rt_inline rt_private_ctx_t _anon_mobj_to_pctx(rt_mem_obj_t mobj)
|
|
{
|
|
return rt_container_of(mobj, struct rt_private_ctx, mem_obj);
|
|
}
|
|
|
|
static long rt_aspace_anon_ref_inc(rt_mem_obj_t aobj)
|
|
{
|
|
long rc;
|
|
if (aobj)
|
|
{
|
|
rc = rt_atomic_add(_anon_obj_get_reference(aobj), 1);
|
|
LOG_D("%s(aobj=%p) Cur %ld", __func__, aobj, rc + 1);
|
|
}
|
|
else
|
|
rc = -1;
|
|
return rc;
|
|
}
|
|
|
|
rt_err_t rt_aspace_anon_ref_dec(rt_mem_obj_t aobj)
|
|
{
|
|
rt_err_t rc;
|
|
rt_aspace_t aspace;
|
|
rt_private_ctx_t pctx;
|
|
long former_reference;
|
|
|
|
if (aobj)
|
|
{
|
|
pctx = _anon_mobj_to_pctx(aobj);
|
|
RT_ASSERT(pctx);
|
|
|
|
former_reference = rt_atomic_add(_anon_obj_get_reference(aobj), -1);
|
|
LOG_D("%s(aobj=%p) Cur %ld", __func__, aobj, former_reference - 1);
|
|
if (pctx->readonly)
|
|
{
|
|
if (former_reference - 1 <= pctx->readonly)
|
|
{
|
|
void *pgtbl;
|
|
RT_ASSERT(former_reference - 1 == pctx->readonly);
|
|
aspace = _anon_obj_get_backup(aobj);
|
|
|
|
pctx->readonly = 0;
|
|
pgtbl = aspace->page_table;
|
|
rt_aspace_delete(aspace);
|
|
rt_hw_mmu_pgtbl_delete(pgtbl);
|
|
}
|
|
}
|
|
else if (former_reference < 2)
|
|
{
|
|
aspace = _anon_obj_get_backup(aobj);
|
|
aspace->private_object = RT_NULL;
|
|
|
|
rt_free(pctx);
|
|
}
|
|
rc = RT_EOK;
|
|
}
|
|
else
|
|
{
|
|
rc = -RT_EINVAL;
|
|
}
|
|
|
|
return rc;
|
|
}
|
|
|
|
void rt_varea_pgmgr_insert(rt_varea_t varea, void *page_addr)
|
|
{
|
|
/* each mapping of page frame in the varea is binding with a reference */
|
|
rt_page_ref_inc(page_addr, 0);
|
|
}
|
|
|
|
/**
|
|
* Private unmapping of address space
|
|
*/
|
|
static void _pgmgr_pop_all(rt_varea_t varea)
|
|
{
|
|
rt_aspace_t aspace = varea->aspace;
|
|
char *iter = varea->start;
|
|
char *end_addr = iter + varea->size;
|
|
|
|
RT_ASSERT(iter < end_addr);
|
|
RT_ASSERT(!((long)iter & ARCH_PAGE_MASK));
|
|
RT_ASSERT(!((long)end_addr & ARCH_PAGE_MASK));
|
|
|
|
for (; iter != end_addr; iter += ARCH_PAGE_SIZE)
|
|
{
|
|
void *page_pa = rt_hw_mmu_v2p(aspace, iter);
|
|
char *page_va = rt_kmem_p2v(page_pa);
|
|
if (page_pa != ARCH_MAP_FAILED && page_va)
|
|
{
|
|
rt_hw_mmu_unmap(aspace, iter, ARCH_PAGE_SIZE);
|
|
rt_pages_free(page_va, 0);
|
|
}
|
|
}
|
|
}
|
|
|
|
static void _pgmgr_pop_range(rt_varea_t varea, void *rm_start, void *rm_end)
|
|
{
|
|
void *page_va;
|
|
|
|
RT_ASSERT(!((rt_ubase_t)rm_start & ARCH_PAGE_MASK));
|
|
RT_ASSERT(!((rt_ubase_t)rm_end & ARCH_PAGE_MASK));
|
|
while (rm_start != rm_end)
|
|
{
|
|
page_va = rt_hw_mmu_v2p(varea->aspace, rm_start);
|
|
|
|
if (page_va != ARCH_MAP_FAILED)
|
|
{
|
|
page_va -= PV_OFFSET;
|
|
LOG_D("%s: free page %p", __func__, page_va);
|
|
rt_varea_unmap_page(varea, rm_start);
|
|
rt_pages_free(page_va, 0);
|
|
}
|
|
rm_start += ARCH_PAGE_SIZE;
|
|
}
|
|
}
|
|
|
|
static const char *_anon_get_name(rt_varea_t varea)
|
|
{
|
|
return varea->aspace == _anon_obj_get_backup(varea->mem_obj) ? "anonymous" : "reference";
|
|
}
|
|
|
|
/**
|
|
* Migration handler on varea re-construction
|
|
*/
|
|
|
|
static void _anon_varea_open(struct rt_varea *varea)
|
|
{
|
|
rt_aspace_anon_ref_inc(varea->mem_obj);
|
|
|
|
if (varea->aspace == _anon_obj_get_backup(varea->mem_obj))
|
|
varea->offset = MM_PA_TO_OFF(varea->start);
|
|
|
|
varea->data = NULL;
|
|
}
|
|
|
|
static void _anon_varea_close(struct rt_varea *varea)
|
|
{
|
|
rt_aspace_anon_ref_dec(varea->mem_obj);
|
|
|
|
/* unmap and dereference page frames in the varea region */
|
|
_pgmgr_pop_all(varea);
|
|
}
|
|
|
|
static rt_err_t _anon_varea_expand(struct rt_varea *varea, void *new_vaddr, rt_size_t size)
|
|
{
|
|
return RT_EOK;
|
|
}
|
|
|
|
static rt_err_t _anon_varea_shrink(rt_varea_t varea, void *new_start, rt_size_t size)
|
|
{
|
|
char *varea_start = varea->start;
|
|
void *rm_start;
|
|
void *rm_end;
|
|
|
|
if (varea_start == (char *)new_start)
|
|
{
|
|
rm_start = varea_start + size;
|
|
rm_end = varea_start + varea->size;
|
|
}
|
|
else /* if (varea_start < (char *)new_start) */
|
|
{
|
|
RT_ASSERT(varea_start < (char *)new_start);
|
|
rm_start = varea_start;
|
|
rm_end = new_start;
|
|
}
|
|
|
|
_pgmgr_pop_range(varea, rm_start, rm_end);
|
|
return RT_EOK;
|
|
}
|
|
|
|
static rt_err_t _anon_varea_split(struct rt_varea *existed, void *unmap_start, rt_size_t unmap_len, struct rt_varea *subset)
|
|
{
|
|
/* remove the resource in the unmap region, and do nothing for the subset */
|
|
_pgmgr_pop_range(existed, unmap_start, (char *)unmap_start + unmap_len);
|
|
|
|
_anon_varea_open(subset);
|
|
return RT_EOK;
|
|
}
|
|
|
|
static rt_err_t _anon_varea_merge(struct rt_varea *merge_to, struct rt_varea *merge_from)
|
|
{
|
|
/* do nothing for the varea merge */
|
|
return RT_EOK;
|
|
}
|
|
|
|
/**
|
|
* Private mapping of address space
|
|
*/
|
|
|
|
rt_inline void _map_page_in_varea(rt_aspace_t asapce, rt_varea_t varea,
|
|
struct rt_aspace_fault_msg *msg, char *fault_addr)
|
|
{
|
|
char *page_va = msg->response.vaddr;
|
|
if (rt_varea_map_page(varea, fault_addr, page_va) == RT_EOK)
|
|
{
|
|
msg->response.status = MM_FAULT_STATUS_OK_MAPPED;
|
|
rt_varea_pgmgr_insert(varea, page_va);
|
|
}
|
|
else
|
|
{
|
|
msg->response.status = MM_FAULT_STATUS_UNRECOVERABLE;
|
|
LOG_W("%s: failed to map page into varea", __func__);
|
|
}
|
|
}
|
|
|
|
/* page frame inquiry or allocation in backup address space */
|
|
static void *_get_page_from_backup(rt_aspace_t backup, rt_base_t offset_in_mobj)
|
|
{
|
|
void *frame_pa;
|
|
char *backup_addr;
|
|
rt_varea_t backup_varea;
|
|
void *rc = RT_NULL;
|
|
|
|
backup_addr = (char *)(offset_in_mobj << MM_PAGE_SHIFT);
|
|
backup_varea = rt_aspace_query(backup, backup_addr);
|
|
|
|
if (backup_varea)
|
|
{
|
|
/* synchronize between multiple request by aspace lock of backup */
|
|
WR_LOCK(backup);
|
|
|
|
frame_pa = rt_hw_mmu_v2p(backup, backup_addr);
|
|
if (frame_pa == ARCH_MAP_FAILED)
|
|
{
|
|
/* provide the page in backup varea */
|
|
struct rt_aspace_fault_msg msg;
|
|
msg.fault_op = MM_FAULT_OP_WRITE;
|
|
msg.fault_type = MM_FAULT_TYPE_PAGE_FAULT;
|
|
msg.fault_vaddr = backup_addr;
|
|
msg.off = offset_in_mobj;
|
|
rt_mm_fault_res_init(&msg.response);
|
|
|
|
rt_mm_dummy_mapper.on_page_fault(backup_varea, &msg);
|
|
if (msg.response.status != MM_FAULT_STATUS_UNRECOVERABLE)
|
|
{
|
|
_map_page_in_varea(backup, backup_varea, &msg, backup_addr);
|
|
if (msg.response.status == MM_FAULT_STATUS_OK_MAPPED)
|
|
{
|
|
rc = msg.response.vaddr;
|
|
}
|
|
rt_pages_free(msg.response.vaddr, 0);
|
|
}
|
|
}
|
|
else
|
|
{
|
|
rc = rt_kmem_p2v(frame_pa);
|
|
if (!rc)
|
|
RT_ASSERT(0 && "No kernel address of target page frame");
|
|
}
|
|
WR_UNLOCK(backup);
|
|
}
|
|
else
|
|
{
|
|
/* out of range error */
|
|
LOG_E("(backup_addr=%p): Page request out of range", backup_addr);
|
|
}
|
|
|
|
return rc;
|
|
}
|
|
|
|
/* get the backup page in kernel for the address in user space */
|
|
static void _fetch_page_for_varea(struct rt_varea *varea, struct rt_aspace_fault_msg *msg, rt_bool_t need_map)
|
|
{
|
|
void *paddr;
|
|
char *frame_ka;
|
|
rt_aspace_t curr_aspace = varea->aspace;
|
|
rt_aspace_t backup = _anon_obj_get_backup(varea->mem_obj);
|
|
|
|
RDWR_LOCK(curr_aspace);
|
|
|
|
/**
|
|
* if the page is already mapped(this may caused by data race while other
|
|
* thread success to take the lock and mapped the page before this), return okay
|
|
*/
|
|
paddr = rt_hw_mmu_v2p(curr_aspace, msg->fault_vaddr);
|
|
if (paddr == ARCH_MAP_FAILED)
|
|
{
|
|
if (backup == curr_aspace)
|
|
{
|
|
rt_mm_dummy_mapper.on_page_fault(varea, msg);
|
|
if (msg->response.status != MM_FAULT_STATUS_UNRECOVERABLE)
|
|
{
|
|
/* if backup == curr_aspace, a page fetch always binding with a pte filling */
|
|
_map_page_in_varea(backup, varea, msg, msg->fault_vaddr);
|
|
if (msg->response.status != MM_FAULT_STATUS_UNRECOVERABLE)
|
|
{
|
|
rt_pages_free(msg->response.vaddr, 0);
|
|
}
|
|
}
|
|
}
|
|
else
|
|
{
|
|
frame_ka = _get_page_from_backup(backup, msg->off);
|
|
if (frame_ka)
|
|
{
|
|
msg->response.vaddr = frame_ka;
|
|
msg->response.size = ARCH_PAGE_SIZE;
|
|
if (!need_map)
|
|
{
|
|
msg->response.status = MM_FAULT_STATUS_OK;
|
|
}
|
|
else
|
|
{
|
|
_map_page_in_varea(curr_aspace, varea, msg, msg->fault_vaddr);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
else
|
|
{
|
|
msg->response.status = MM_FAULT_STATUS_OK_MAPPED;
|
|
}
|
|
RDWR_UNLOCK(curr_aspace);
|
|
}
|
|
|
|
static void _anon_page_fault(struct rt_varea *varea, struct rt_aspace_fault_msg *msg)
|
|
{
|
|
_fetch_page_for_varea(varea, msg, RT_TRUE);
|
|
}
|
|
|
|
static void read_by_mte(rt_aspace_t aspace, struct rt_aspace_io_msg *iomsg)
|
|
{
|
|
if (rt_aspace_page_get_phy(aspace, iomsg->fault_vaddr, iomsg->buffer_vaddr) == RT_EOK)
|
|
iomsg->response.status = MM_FAULT_STATUS_OK;
|
|
}
|
|
|
|
static void _anon_page_read(struct rt_varea *varea, struct rt_aspace_io_msg *iomsg)
|
|
{
|
|
rt_aspace_t curr_aspace = varea->aspace;
|
|
rt_aspace_t backup = _anon_obj_get_backup(varea->mem_obj);
|
|
|
|
if (rt_hw_mmu_v2p(curr_aspace, iomsg->fault_vaddr) == ARCH_MAP_FAILED)
|
|
{
|
|
struct rt_aspace_fault_msg msg;
|
|
msg.fault_op = MM_FAULT_OP_READ;
|
|
msg.fault_type = MM_FAULT_TYPE_PAGE_FAULT;
|
|
msg.fault_vaddr = iomsg->fault_vaddr;
|
|
msg.off = iomsg->off;
|
|
rt_mm_fault_res_init(&msg.response);
|
|
|
|
_fetch_page_for_varea(varea, &msg, RT_FALSE);
|
|
if (msg.response.status != MM_FAULT_STATUS_UNRECOVERABLE)
|
|
{
|
|
void *saved_fault_va = iomsg->fault_vaddr;
|
|
iomsg->fault_vaddr = (void *)(iomsg->off << MM_PAGE_SHIFT);
|
|
read_by_mte(backup, iomsg);
|
|
iomsg->fault_vaddr = saved_fault_va;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
read_by_mte(curr_aspace, iomsg);
|
|
}
|
|
}
|
|
|
|
static void write_by_mte(rt_aspace_t aspace, struct rt_aspace_io_msg *iomsg)
|
|
{
|
|
if (rt_aspace_page_put_phy(aspace, iomsg->fault_vaddr, iomsg->buffer_vaddr) == RT_EOK)
|
|
iomsg->response.status = MM_FAULT_STATUS_OK;
|
|
}
|
|
|
|
static void _anon_page_write(struct rt_varea *varea, struct rt_aspace_io_msg *iomsg)
|
|
{
|
|
rt_aspace_t from_aspace = varea->aspace;
|
|
rt_aspace_t backup = _anon_obj_get_backup(varea->mem_obj);
|
|
|
|
if (from_aspace != backup)
|
|
{
|
|
/* varea in guest aspace cannot modify the page */
|
|
iomsg->response.status = MM_FAULT_STATUS_UNRECOVERABLE;
|
|
}
|
|
else if (rt_hw_mmu_v2p(from_aspace, iomsg->fault_vaddr) == ARCH_MAP_FAILED)
|
|
{
|
|
struct rt_aspace_fault_msg msg;
|
|
msg.fault_op = MM_FAULT_OP_WRITE;
|
|
msg.fault_type = MM_FAULT_TYPE_PAGE_FAULT;
|
|
msg.fault_vaddr = iomsg->fault_vaddr;
|
|
msg.off = iomsg->off;
|
|
rt_mm_fault_res_init(&msg.response);
|
|
|
|
_fetch_page_for_varea(varea, &msg, RT_TRUE);
|
|
if (msg.response.status == MM_FAULT_STATUS_OK_MAPPED)
|
|
{
|
|
write_by_mte(backup, iomsg);
|
|
}
|
|
else
|
|
{
|
|
/* mapping failed, report an error */
|
|
iomsg->response.status = MM_FAULT_STATUS_UNRECOVERABLE;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
write_by_mte(backup, iomsg);
|
|
}
|
|
}
|
|
|
|
static struct rt_private_ctx _priv_obj = {
|
|
.mem_obj.get_name = _anon_get_name,
|
|
.mem_obj.on_page_fault = _anon_page_fault,
|
|
.mem_obj.hint_free = NULL,
|
|
.mem_obj.on_varea_open = _anon_varea_open,
|
|
.mem_obj.on_varea_close = _anon_varea_close,
|
|
.mem_obj.on_varea_shrink = _anon_varea_shrink,
|
|
.mem_obj.on_varea_split = _anon_varea_split,
|
|
.mem_obj.on_varea_expand = _anon_varea_expand,
|
|
.mem_obj.on_varea_merge = _anon_varea_merge,
|
|
.mem_obj.page_read = _anon_page_read,
|
|
.mem_obj.page_write = _anon_page_write,
|
|
};
|
|
|
|
rt_inline rt_private_ctx_t rt_private_obj_create_n_bind(rt_aspace_t aspace)
|
|
{
|
|
rt_private_ctx_t private_object;
|
|
private_object = rt_malloc(sizeof(struct rt_private_ctx));
|
|
if (private_object)
|
|
{
|
|
memcpy(&private_object->mem_obj, &_priv_obj, sizeof(_priv_obj));
|
|
|
|
/* hold a init ref from backup aspace */
|
|
rt_atomic_store(&private_object->reference, 1);
|
|
|
|
private_object->readonly = RT_FALSE;
|
|
private_object->backup_aspace = aspace;
|
|
aspace->private_object = &private_object->mem_obj;
|
|
}
|
|
|
|
return private_object;
|
|
}
|
|
|
|
rt_inline rt_mem_obj_t _get_private_obj(rt_aspace_t aspace)
|
|
{
|
|
rt_private_ctx_t priv;
|
|
rt_mem_obj_t rc;
|
|
rc = aspace->private_object;
|
|
if (!aspace->private_object)
|
|
{
|
|
priv = rt_private_obj_create_n_bind(aspace);
|
|
if (priv)
|
|
{
|
|
rc = &priv->mem_obj;
|
|
aspace->private_object = rc;
|
|
}
|
|
}
|
|
return rc;
|
|
}
|
|
|
|
static int _override_map(rt_varea_t varea, rt_aspace_t aspace, void *fault_vaddr, struct rt_aspace_fault_msg *msg, void *page)
|
|
{
|
|
int rc = MM_FAULT_FIXABLE_FALSE;
|
|
rt_mem_obj_t private_object;
|
|
rt_varea_t map_varea = RT_NULL;
|
|
rt_err_t error;
|
|
rt_size_t flags;
|
|
rt_size_t attr;
|
|
|
|
LOG_D("%s", __func__);
|
|
|
|
private_object = _get_private_obj(aspace);
|
|
|
|
if (private_object)
|
|
{
|
|
flags = varea->flag | MMF_MAP_FIXED;
|
|
/* don't prefetch and do it latter */
|
|
flags &= ~MMF_PREFETCH;
|
|
attr = rt_hw_mmu_attr_add_perm(varea->attr, RT_HW_MMU_PROT_USER | RT_HW_MMU_PROT_WRITE);
|
|
|
|
/* override existing mapping at fault_vaddr */
|
|
error = _mm_aspace_map(
|
|
aspace, &map_varea, &fault_vaddr, ARCH_PAGE_SIZE, attr,
|
|
flags, private_object, MM_PA_TO_OFF(fault_vaddr));
|
|
|
|
if (error == RT_EOK)
|
|
{
|
|
msg->response.status = MM_FAULT_STATUS_OK;
|
|
msg->response.vaddr = page;
|
|
msg->response.size = ARCH_PAGE_SIZE;
|
|
if (rt_varea_map_with_msg(map_varea, msg) != RT_EOK)
|
|
{
|
|
LOG_E("%s: fault_va=%p,(priv_va=%p,priv_sz=0x%lx) at %s", __func__, msg->fault_vaddr, map_varea->start, map_varea->size, VAREA_NAME(map_varea));
|
|
RT_ASSERT(0 && "should never failed");
|
|
}
|
|
RT_ASSERT(rt_hw_mmu_v2p(aspace, msg->fault_vaddr) == (page + PV_OFFSET));
|
|
rc = MM_FAULT_FIXABLE_TRUE;
|
|
rt_varea_pgmgr_insert(map_varea, page);
|
|
rt_pages_free(page, 0);
|
|
}
|
|
else
|
|
{
|
|
/* private object will be release on destruction of aspace */
|
|
rt_free(map_varea);
|
|
}
|
|
}
|
|
else
|
|
{
|
|
LOG_I("%s: out of memory", __func__);
|
|
rc = MM_FAULT_FIXABLE_FALSE;
|
|
}
|
|
|
|
return rc;
|
|
}
|
|
|
|
/**
|
|
* replace an existing mapping to a private one, this is identical to:
|
|
* => aspace_unmap(ex_varea, )
|
|
* => aspace_map()
|
|
*/
|
|
int rt_varea_fix_private_locked(rt_varea_t ex_varea, void *pa,
|
|
struct rt_aspace_fault_msg *msg,
|
|
rt_bool_t dont_copy)
|
|
{
|
|
/**
|
|
* todo: READ -> WRITE lock here
|
|
*/
|
|
void *page;
|
|
void *fault_vaddr;
|
|
rt_aspace_t aspace;
|
|
rt_mem_obj_t ex_obj;
|
|
int rc = MM_FAULT_FIXABLE_FALSE;
|
|
ex_obj = ex_varea->mem_obj;
|
|
|
|
if (ex_obj)
|
|
{
|
|
fault_vaddr = msg->fault_vaddr;
|
|
aspace = ex_varea->aspace;
|
|
RT_ASSERT(!!aspace);
|
|
|
|
/**
|
|
* todo: what if multiple pages are required?
|
|
*/
|
|
if (aspace->private_object == ex_obj)
|
|
{
|
|
RT_ASSERT(0 && "recursion");
|
|
}
|
|
else if (ex_obj->page_read)
|
|
{
|
|
page = rt_pages_alloc_ext(0, PAGE_ANY_AVAILABLE);
|
|
if (page)
|
|
{
|
|
/** setup message & fetch the data from source object */
|
|
if (!dont_copy)
|
|
{
|
|
struct rt_aspace_io_msg io_msg;
|
|
rt_mm_io_msg_init(&io_msg, msg->off, msg->fault_vaddr, page);
|
|
ex_obj->page_read(ex_varea, &io_msg);
|
|
/**
|
|
* Note: if ex_obj have mapped into varea, it's still okay since
|
|
* we will override it latter
|
|
*/
|
|
if (io_msg.response.status != MM_FAULT_STATUS_UNRECOVERABLE)
|
|
{
|
|
rc = _override_map(ex_varea, aspace, fault_vaddr, msg, page);
|
|
}
|
|
else
|
|
{
|
|
rt_pages_free(page, 0);
|
|
LOG_I("%s: page read(va=%p) fault from %s(start=%p,size=%p)", __func__,
|
|
msg->fault_vaddr, VAREA_NAME(ex_varea), ex_varea->start, ex_varea->size);
|
|
}
|
|
}
|
|
else
|
|
{
|
|
rc = _override_map(ex_varea, aspace, fault_vaddr, msg, page);
|
|
}
|
|
}
|
|
else
|
|
{
|
|
LOG_I("%s: pages allocation failed", __func__);
|
|
}
|
|
}
|
|
else
|
|
{
|
|
LOG_I("%s: no page read method provided from %s", __func__, VAREA_NAME(ex_varea));
|
|
}
|
|
}
|
|
else
|
|
{
|
|
LOG_I("%s: unavailable memory object", __func__);
|
|
}
|
|
|
|
return rc;
|
|
}
|
|
|
|
int rt_aspace_map_private(rt_aspace_t aspace, void **addr, rt_size_t length,
|
|
rt_size_t attr, mm_flag_t flags)
|
|
{
|
|
int rc;
|
|
rt_mem_obj_t priv_obj;
|
|
|
|
if (flags & MMF_STATIC_ALLOC)
|
|
{
|
|
rc = -RT_EINVAL;
|
|
}
|
|
else
|
|
{
|
|
priv_obj = _get_private_obj(aspace);
|
|
if (priv_obj)
|
|
{
|
|
flags |= MMF_MAP_PRIVATE;
|
|
flags &= ~MMF_PREFETCH;
|
|
rc = rt_aspace_map(aspace, addr, length, attr, flags, priv_obj, 0);
|
|
}
|
|
else
|
|
{
|
|
rc = -RT_ENOMEM;
|
|
}
|
|
}
|
|
return rc;
|
|
}
|
|
|
|
static int _release_shared(rt_varea_t varea, void *arg)
|
|
{
|
|
rt_aspace_t src = varea->aspace;
|
|
rt_mem_obj_t mem_obj = varea->mem_obj;
|
|
|
|
if (mem_obj != _get_private_obj(src))
|
|
{
|
|
_varea_uninstall_locked(varea);
|
|
if (VAREA_NOT_STATIC(varea))
|
|
{
|
|
rt_free(varea);
|
|
}
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
static rt_err_t _convert_readonly(rt_aspace_t aspace, long base_reference)
|
|
{
|
|
rt_mem_obj_t aobj;
|
|
rt_private_ctx_t pctx;
|
|
aobj = _get_private_obj(aspace);
|
|
pctx = _anon_mobj_to_pctx(aobj);
|
|
|
|
LOG_D("Ref(cur=%d,base=%d)", pctx->reference, base_reference);
|
|
rt_aspace_traversal(aspace, _release_shared, 0);
|
|
pctx->readonly = base_reference;
|
|
return 0;
|
|
}
|
|
|
|
rt_inline void _switch_aspace(rt_aspace_t *pa, rt_aspace_t *pb)
|
|
{
|
|
rt_aspace_t temp;
|
|
temp = *pa;
|
|
*pa = *pb;
|
|
*pb = temp;
|
|
}
|
|
|
|
rt_err_t rt_aspace_fork(rt_aspace_t *psrc, rt_aspace_t *pdst)
|
|
{
|
|
rt_err_t rc;
|
|
void *pgtbl;
|
|
rt_aspace_t backup;
|
|
rt_aspace_t src = *psrc;
|
|
rt_aspace_t dst = *pdst;
|
|
long base_reference;
|
|
|
|
pgtbl = rt_hw_mmu_pgtbl_create();
|
|
if (pgtbl)
|
|
{
|
|
backup = rt_aspace_create(src->start, src->size, pgtbl);
|
|
if (backup)
|
|
{
|
|
WR_LOCK(src);
|
|
base_reference = rt_atomic_load(_anon_obj_get_reference(src->private_object));
|
|
rc = rt_aspace_duplicate_locked(src, dst);
|
|
WR_UNLOCK(src);
|
|
|
|
if (!rc)
|
|
{
|
|
/* WR_LOCK(dst) is not necessary since dst is not available currently */
|
|
rc = rt_aspace_duplicate_locked(dst, backup);
|
|
if (!rc)
|
|
{
|
|
_switch_aspace(psrc, &backup);
|
|
_convert_readonly(backup, base_reference);
|
|
}
|
|
}
|
|
}
|
|
else
|
|
{
|
|
rc = -RT_ENOMEM;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
rc = -RT_ENOMEM;
|
|
}
|
|
|
|
return rc;
|
|
}
|