Skip to content

L
linux-elphel
Commit 531d1367 authored by Andrey Filippov's avatar Andrey Filippov
Browse files
Options

trying allocating only one page to pass phys. addr

parent 09fa190d
Show whitespace changes
Inline Side-by-side
Showing with 118 additions and 34 deletions
src/block/bio.c
View file @ 531d1367
......@@ -843,7 +843,7 @@ EXPORT_SYMBOL(bio_add_page);
* the next non-empty segment of the iov iterator.
*/
// undefine to skip
#define SUSPECT_MMAPED (4 * PAGE_SIZE)
//#define ELPHEL_SUSPECT_MMAPED (4 * PAGE_SIZE)
static int __bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter)
{
unsigned short nr_pages = bio->bi_max_vecs - bio->bi_vcnt, idx;
......@@ -851,6 +851,7 @@ static int __bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter)
struct page **pages = (struct page **)bv;
size_t offset;
ssize_t size;
int single_page = 0;
if (bio->bi_disk && (bio->bi_disk->major == 8)) { // sda, sda*
pr_debug(" *A* nr_pages=0x%04x,bio->bi_max_vecs=0x%04x, bio->bi_vcnt=0x%04x", nr_pages, bio->bi_max_vecs, bio->bi_vcnt); // *A* nr_pages=0x009f,bio->bi_max_vecs=0x009f, bio->bi_vcnt==0x0000
pr_debug(" *A0* max_hw_sectors=%d, max_dev_sectors=%d, max_sectors=%d, cluster=%d, max_segment_size=0x%08x",\
......@@ -872,24 +873,33 @@ fail.
pr_debug(" *B* niter->type=%d, iter->iov_offset=0x%08x, iter->count=0x%08x, iter->nr_segs=0x%08lx",\
iter->type, iter->iov_offset,iter->count,iter->nr_segs); // *B* niter->type=1, iter->iov_offset=0x00000000, iter->count=0x0009f000, iter->nr_segs=0x00000001
}
#ifdef ELPHEL_MMAP_PAGED
// If suspected "Elphel" - try it first?
#ifdef SUSPECT_MMAPED
if (( bio->bi_vcnt == 0) && (iter->iov_offset == 0) && (iter->nr_segs == 1) && (iter->count >= SUSPECT_MMAPED)) {
pr_debug(" *B1* Suspecting mmaped memory, iter->count=0x%08x > 0x%08x", iter->count, (int) SUSPECT_MMAPED);
#ifdef ELPHEL_SUSPECT_MMAPED
if (( bio->bi_vcnt == 0) && (iter->iov_offset == 0) && (iter->nr_segs == 1) && (iter->count >= ELPHEL_SUSPECT_MMAPED)) {
pr_debug(" *B1* Suspecting mmaped memory, iter->count=0x%08x > 0x%08x", iter->count, (int) ELPHEL_SUSPECT_MMAPED);
size= iov_iter_get_pages_elphel(iter, pages, LONG_MAX, nr_pages, &offset); // try mmap-ed
if (size < 0){
pr_debug("mmaped failed (err=%d) trying pinning userspace memory", size);
size = iov_iter_get_pages(iter, pages, LONG_MAX, nr_pages, &offset); // regular way with userspace
#ifdef ELPHEL_MMAP_SINGLE_PAGE
} else {
single_page = 1;
#endif
}
} else {
size = iov_iter_get_pages(iter, pages, LONG_MAX, nr_pages, &offset); // regular way with userspace
}
#else
size = iov_iter_get_pages(iter, pages, LONG_MAX, nr_pages, &offset);
#endif
#else
size = iov_iter_get_pages(iter, pages, LONG_MAX, nr_pages, &offset);
#endif
if (unlikely(size <= 0)) {
pr_debug("Error=%d, trying continuous mmap-ed pages, bio->bi_vcnt=0x%04x, bio->bi_max_vecs=0x%04x, offset=0x%08x",
size, bio->bi_vcnt, bio->bi_max_vecs, offset);
#ifdef ELPHEL_MMAP_PAGED
if ((size == -EFAULT) && ( bio->bi_vcnt == 0) && (iter->iov_offset == 0) && (iter->nr_segs == 1)){ // Only for all pages, single segment
size= iov_iter_get_pages_elphel(iter, pages, LONG_MAX, nr_pages, &offset);
}
......@@ -897,11 +907,29 @@ fail.
pr_debug("Error after iov_iter_get_pages_elphel: %d, trying continuous mmap-ed pages, bio->bi_vcnt=0x%04x, bio->bi_max_vecs=0x%04x, offset=0x%08x",
size, bio->bi_vcnt, bio->bi_max_vecs, offset);
return size ? size : -EFAULT;
#ifdef ELPHEL_MMAP_SINGLE_PAGE
} else {
single_page = 1;
#endif
}
#else
return size ? size : -EFAULT;
#endif
}
if (single_page) {
nr_pages = 1;
bio->bi_iter.bi_size += size;
bio->bi_vcnt += nr_pages;
bv[0].bv_page = pages[idx];
bv[0].bv_len = size;
bv[0].bv_offset = offset;
if (bio->bi_disk && (bio->bi_disk->major == 8)) { // sda, sda*
pr_debug(" *C1* nr_pages=0x%04x, size=0x%08x, offset==0x%04x", nr_pages, size, offset); // *C* nr_pages=0x009f, size=0x0009f000, offset==0x0000
}
} else {
idx = nr_pages = (size + offset + PAGE_SIZE - 1) / PAGE_SIZE;
if (bio->bi_disk && (bio->bi_disk->major == 8)) { // sda, sda*
pr_debug(" *C* nr_pages=0x%04x, size=0x%08x, offset==0x%04x", nr_pages, size, offset); // *C* nr_pages=0x009f, size=0x0009f000, offset==0x0000
pr_debug(" *C0* nr_pages=0x%04x, size=0x%08x, offset==0x%04x", nr_pages, size, offset); // *C* nr_pages=0x009f, size=0x0009f000, offset==0x0000
}
/*
* Deep magic below: We need to walk the pinned pages backwards
......@@ -923,6 +951,7 @@ fail.
bv[0].bv_offset += offset;
bv[0].bv_len -= offset;
bv[nr_pages - 1].bv_len -= nr_pages * PAGE_SIZE - offset - size;
}
if (bio->bi_disk && (bio->bi_disk->major == 8)) { // sda, sda* *D* nr_pages=0x00a0, size=0x000a0000, offset=0x0000, iter->count=0x000a0000, iter->nr_segs=0x00000001
pr_debug(" *D* nr_pages=0x%04x, size=0x%08x, offset=0x%04x, iter->count=0x%08x, iter->nr_segs=0x%08lx", nr_pages, size, offset, iter->count, iter->nr_segs);
}
......@@ -932,7 +961,6 @@ fail.
iter->type, iter->iov_offset, iter->count, iter->nr_segs);
pr_debug(" *F* nr_pages=0x%04x, size=0x%08x, offset==0x%04x", nr_pages, size, offset); // *F* nr_pages=0x009f, size=0x0009f000, offset==0x0000
}
return 0;
}
......@@ -963,6 +991,48 @@ int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter)
}
EXPORT_SYMBOL_GPL(bio_iov_iter_get_pages);
int bio_iov_iter_get_pages_elphel(struct bio *bio, struct iov_iter *iter)
{
int indx, n; // page index
// size_t offset;
// ssize_t size;
size_t len;
struct bio_vec *bv = bio->bi_io_vec + bio->bi_vcnt; // bio->bi_vcnt should be 0
const struct iovec *iov = iter->iov; // only use first in array - supposed to be a single-element
unsigned long paddr = 0; // physical address where to write
unsigned long addr;
unsigned long vaddr = (unsigned long) iov->iov_base;
struct vm_area_struct *vm_area;
if (iter->nr_segs != 1){
pr_debug("nr_segs = %d != 1", iter->nr_segs);
return -EINVAL;
}
// pr_debug("sizeof(struct vm_area_struct)=%d", sizeof(struct vm_area_struct));
for (vm_area = current->mm->mmap; vm_area; vm_area = vm_area-> vm_next) {
if ((vaddr >= vm_area->vm_start) && (vaddr < vm_area->vm_end)) {
pr_debug("physical address = 0x%08lx", (unsigned long) vm_area->vm_private_data);
print_hex_dump_bytes("", DUMP_PREFIX_OFFSET, vm_area, sizeof(struct vm_area_struct));
paddr = (unsigned long) vm_area->vm_private_data;
if (paddr) {
paddr += vaddr - vm_area->vm_start; // added offset within buffer
}
break;
}
}
if (!paddr){
pr_debug("mmap-ed physical address is not specified in vm_area->vm_private_data, aborting");
return -EINVAL;
}
pr_debug("virtual address = 0x%08lx, physical address = 0x%08lx", vaddr, paddr);
bv[0].bv_page = NULL; // Make sure it will not be tried to release
bv[0].bv_len = iov->iov_len;
bv[0].bv_offset = 0;
}
static void submit_bio_wait_endio(struct bio *bio)
{
......
src/block/blk-core.c
View file @ 531d1367
......@@ -2466,7 +2466,8 @@ blk_qc_t generic_make_request(struct bio *bio)
}
}
if (bio->bi_disk && (bio->bi_disk->major == 8)) { // sda, sda*
pr_debug(" *{* bio->bi_flags=0x%04x, bio->bi_phys_segments=0x%08x, enter_succeeded=%d",bio->bi_flags, bio->bi_phys_segments, enter_succeeded);
pr_debug(" *{* state before blk_mq_make_request(): bio->bi_flags=0x%04x, bio->bi_phys_segments=0x%08x, enter_succeeded=%d",bio->bi_flags, bio->bi_phys_segments, enter_succeeded);
print_hex_dump_bytes("", DUMP_PREFIX_OFFSET, bio, sizeof(struct bio));
}
if (enter_succeeded) {
struct bio_list lower, same;
......@@ -2474,7 +2475,7 @@ blk_qc_t generic_make_request(struct bio *bio)
/* Create a fresh bio_list for all subordinate requests */
bio_list_on_stack[1] = bio_list_on_stack[0];
bio_list_init(&bio_list_on_stack[0]);
ret = q->make_request_fn(q, bio);
ret = q->make_request_fn(q, bio); // in this case -> blk_mq_make_request()
if (bio->bi_disk && (bio->bi_disk->major == 8)) { // sda, sda*
pr_debug("q->make_request_fn(q, bio) -> %u", ret);
}
......
src/block/blk-merge.c
View file @ 531d1367
......@@ -238,6 +238,7 @@ void blk_queue_split(struct request_queue *q, struct bio **bio)
split = blk_bio_write_same_split(q, *bio, &q->bio_split, &nsegs);
break;
default:
// *** INCREASE q->limits.max_segment_size, it is set from BLK_MAX_SEGMENT_SIZE = 65536 (include/linux/blkdev.h:1429:) Elphel 06/14/2023
split = blk_bio_segment_split(q, *bio, &q->bio_split, &nsegs);
break;
}
......
src/block/blk-mq.c
View file @ 531d1367
......@@ -1830,8 +1830,8 @@ static blk_qc_t blk_mq_make_request(struct request_queue *q, struct bio *bio)
if (bio->bi_disk && (bio->bi_disk->major == 8)) { // sda, sda*
pr_debug(" *1* bio->bi_flags=0x%04x, bio->bi_phys_segments=0x%08x",bio->bi_flags, bio->bi_phys_segments); // not yet set
}
// *** INCREASE q->limits.max_segment_size, it is set from BLK_MAX_SEGMENT_SIZE = 65536 (include/linux/blkdev.h:1429:) Elphel 06/14/2023
blk_queue_split(q, &bio);
if (bio->bi_disk && (bio->bi_disk->major == 8)) { // sda, sda* already split - 0xa segments (16 pages each)
pr_debug(" *2* bio->bi_flags=0x%04x, bio->bi_phys_segments=0x%08x",bio->bi_flags, bio->bi_phys_segments); // already set
}
......@@ -2592,7 +2592,7 @@ struct request_queue *blk_mq_init_allocated_queue(struct blk_mq_tag_set *set,
struct request_queue *q)
{
// if (bio->bi_disk && (bio->bi_disk->major == 8)) { // sda, sda*
pr_debug(" *-* ");
pr_debug(" *-* "); // never
// }
/* mark the queue as mq asap */
q->mq_ops = set->ops;
......
src/fs/block_dev.c
View file @ 531d1367
......@@ -190,7 +190,7 @@ __blkdev_direct_IO_simple(struct kiocb *iocb, struct iov_iter *iter,
{
struct file *file = iocb->ki_filp;
struct block_device *bdev = I_BDEV(bdev_file_inode(file));
struct bio_vec inline_vecs[DIO_INLINE_BIO_VECS], *vecs, *bvec;
struct bio_vec inline_vecs[DIO_INLINE_BIO_VECS], *vecs, *bvec; // Elphel: may reduce it again
loff_t pos = iocb->ki_pos;
bool should_dirty = false;
struct bio bio;
......@@ -222,7 +222,8 @@ __blkdev_direct_IO_simple(struct kiocb *iocb, struct iov_iter *iter,
bio.bi_private = current;
bio.bi_end_io = blkdev_bio_end_io_simple;
bio.bi_ioprio = iocb->ki_ioprio;
ret = bio_iov_iter_get_pages(&bio, iter);
ret = bio_iov_iter_get_pages(&bio, iter); // normally returns 0
pr_debug("ret = %d, bio.bi_vcnt=%d", ret, bio.bi_vcnt); // -14
if (unlikely(ret))
goto out;
......@@ -236,6 +237,7 @@ __blkdev_direct_IO_simple(struct kiocb *iocb, struct iov_iter *iter,
bio.bi_opf = dio_bio_write_op(iocb);
task_io_account_write(ret);
}
pr_debug("bio:");
print_hex_dump_bytes("", DUMP_PREFIX_OFFSET, &bio, sizeof(struct bio));
// pr_debug("bi_disk=0x%p, bi_sector=%llu, bi_size=%u, bi_private=0x%p, bi_vcnt=%u, bi_max_vecs=%u, bi_io_vec=0x%p, bi_pool=0x%p",
// bio.bi_disk, bio.bi_iter.bi_sector, bio.bi_iter.bi_size, bio.bi_private, bio.bi_vcnt, bio.bi_max_vecs, bio.bi_io_vec, bio.bi_pool);
......@@ -250,8 +252,14 @@ __blkdev_direct_IO_simple(struct kiocb *iocb, struct iov_iter *iter,
}
}
pr_debug("ARCH_PFN_OFFSET =0x%08lx, PHYS_PFN_OFFSET=0x%08lx, mem_map=0x%08x, bi_phys_segments=0x%08x", \
ARCH_PFN_OFFSET, PHYS_PFN_OFFSET, (int) mem_map, bio.bi_phys_segments);
ARCH_PFN_OFFSET, PHYS_PFN_OFFSET, (int) mem_map, bio.bi_phys_segments); // bi_phys_segments==0
qc = submit_bio(&bio);
pr_debug("After qc = submit_bio(&bio), bio:");
print_hex_dump_bytes("", DUMP_PREFIX_OFFSET, &bio, sizeof(struct bio));
for (;;) {
set_current_state(TASK_UNINTERRUPTIBLE);
if (!READ_ONCE(bio.bi_private))
......
src/lib/iov_iter.c
View file @ 531d1367
......@@ -1263,7 +1263,11 @@ ssize_t iov_iter_get_pages_elphel(struct iov_iter *i,
if (len > maxpages * PAGE_SIZE)
len = maxpages * PAGE_SIZE;
addr &= ~(PAGE_SIZE - 1);
#ifdef ELPHEL_MMAP_SINGLE_PAGE
n = 1; // only pin a single (first) page
#else
n = DIV_ROUND_UP(len, PAGE_SIZE);
#endif
for (indx = 0; indx < n; indx++){
if ((cur_page -> compound_head) & 1){
pr_debug("**** COMPOUND HEAD! ****");
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment