trying allocating only one page to pass phys. addr (531d1367) · Commits · Elphel / linux-elphel

src/block/bio.c

+97 −27

Original line number	Diff line number	Diff line
		@@ -843,7 +843,7 @@ EXPORT_SYMBOL(bio_add_page);
		* the next non-empty segment of the iov iterator.
		*/
		// undefine to skip
		#define SUSPECT_MMAPED (4 * PAGE_SIZE)
		//#define ELPHEL_SUSPECT_MMAPED (4 * PAGE_SIZE)
		static int __bio_iov_iter_get_pages(struct bio bio, struct iov_iter iter)
		{
		unsigned short nr_pages = bio->bi_max_vecs - bio->bi_vcnt, idx;
		@@ -851,6 +851,7 @@ static int __bio_iov_iter_get_pages(struct bio bio, struct iov_iter iter)
		struct page pages = (struct page )bv;
		size_t offset;
		ssize_t size;
		int single_page = 0;
		if (bio->bi_disk && (bio->bi_disk->major == 8)) { // sda, sda*
		pr_debug(" A nr_pages=0x%04x,bio->bi_max_vecs=0x%04x, bio->bi_vcnt=0x%04x", nr_pages, bio->bi_max_vecs, bio->bi_vcnt); // A nr_pages=0x009f,bio->bi_max_vecs=0x009f, bio->bi_vcnt==0x0000
		pr_debug(" A0 max_hw_sectors=%d, max_dev_sectors=%d, max_sectors=%d, cluster=%d, max_segment_size=0x%08x",\
		@@ -872,24 +873,33 @@ fail.
		pr_debug(" B niter->type=%d, iter->iov_offset=0x%08x, iter->count=0x%08x, iter->nr_segs=0x%08lx",\
		iter->type, iter->iov_offset,iter->count,iter->nr_segs); // B niter->type=1, iter->iov_offset=0x00000000, iter->count=0x0009f000, iter->nr_segs=0x00000001
		}
		#ifdef ELPHEL_MMAP_PAGED
		// If suspected "Elphel" - try it first?
		#ifdef SUSPECT_MMAPED
		if (( bio->bi_vcnt == 0) && (iter->iov_offset == 0) && (iter->nr_segs == 1) && (iter->count >= SUSPECT_MMAPED)) {
		pr_debug(" B1 Suspecting mmaped memory, iter->count=0x%08x > 0x%08x", iter->count, (int) SUSPECT_MMAPED);
		#ifdef ELPHEL_SUSPECT_MMAPED
		if (( bio->bi_vcnt == 0) && (iter->iov_offset == 0) && (iter->nr_segs == 1) && (iter->count >= ELPHEL_SUSPECT_MMAPED)) {
		pr_debug(" B1 Suspecting mmaped memory, iter->count=0x%08x > 0x%08x", iter->count, (int) ELPHEL_SUSPECT_MMAPED);
		size= iov_iter_get_pages_elphel(iter, pages, LONG_MAX, nr_pages, &offset); // try mmap-ed
		if (size < 0){
		pr_debug("mmaped failed (err=%d) trying pinning userspace memory", size);
		size = iov_iter_get_pages(iter, pages, LONG_MAX, nr_pages, &offset); // regular way with userspace
		#ifdef ELPHEL_MMAP_SINGLE_PAGE
		} else {
		single_page = 1;
		#endif
		}
		} else {
		size = iov_iter_get_pages(iter, pages, LONG_MAX, nr_pages, &offset); // regular way with userspace
		}
		#else
		size = iov_iter_get_pages(iter, pages, LONG_MAX, nr_pages, &offset);
		#endif
		#else
		size = iov_iter_get_pages(iter, pages, LONG_MAX, nr_pages, &offset);
		#endif
		if (unlikely(size <= 0)) {
		pr_debug("Error=%d, trying continuous mmap-ed pages, bio->bi_vcnt=0x%04x, bio->bi_max_vecs=0x%04x, offset=0x%08x",
		size, bio->bi_vcnt, bio->bi_max_vecs, offset);
		#ifdef ELPHEL_MMAP_PAGED
		if ((size == -EFAULT) && ( bio->bi_vcnt == 0) && (iter->iov_offset == 0) && (iter->nr_segs == 1)){ // Only for all pages, single segment
		size= iov_iter_get_pages_elphel(iter, pages, LONG_MAX, nr_pages, &offset);
		}
		@@ -897,11 +907,29 @@ fail.
		pr_debug("Error after iov_iter_get_pages_elphel: %d, trying continuous mmap-ed pages, bio->bi_vcnt=0x%04x, bio->bi_max_vecs=0x%04x, offset=0x%08x",
		size, bio->bi_vcnt, bio->bi_max_vecs, offset);
		return size ? size : -EFAULT;
		#ifdef ELPHEL_MMAP_SINGLE_PAGE
		} else {
		single_page = 1;
		#endif
		}
		#else
		return size ? size : -EFAULT;
		#endif
		}
		if (single_page) {
		nr_pages = 1;
		bio->bi_iter.bi_size += size;
		bio->bi_vcnt += nr_pages;
		bv[0].bv_page = pages[idx];
		bv[0].bv_len = size;
		bv[0].bv_offset = offset;
		if (bio->bi_disk && (bio->bi_disk->major == 8)) { // sda, sda*
		pr_debug(" C1 nr_pages=0x%04x, size=0x%08x, offset==0x%04x", nr_pages, size, offset); // C nr_pages=0x009f, size=0x0009f000, offset==0x0000
		}
		} else {
		idx = nr_pages = (size + offset + PAGE_SIZE - 1) / PAGE_SIZE;
		if (bio->bi_disk && (bio->bi_disk->major == 8)) { // sda, sda*
		pr_debug(" C nr_pages=0x%04x, size=0x%08x, offset==0x%04x", nr_pages, size, offset); // C nr_pages=0x009f, size=0x0009f000, offset==0x0000
		pr_debug(" C0 nr_pages=0x%04x, size=0x%08x, offset==0x%04x", nr_pages, size, offset); // C nr_pages=0x009f, size=0x0009f000, offset==0x0000
		}
		/*
		* Deep magic below: We need to walk the pinned pages backwards
		@@ -923,6 +951,7 @@ fail.
		bv[0].bv_offset += offset;
		bv[0].bv_len -= offset;
		bv[nr_pages - 1].bv_len -= nr_pages * PAGE_SIZE - offset - size;
		}
		if (bio->bi_disk && (bio->bi_disk->major == 8)) { // sda, sda* D nr_pages=0x00a0, size=0x000a0000, offset=0x0000, iter->count=0x000a0000, iter->nr_segs=0x00000001
		pr_debug(" D nr_pages=0x%04x, size=0x%08x, offset=0x%04x, iter->count=0x%08x, iter->nr_segs=0x%08lx", nr_pages, size, offset, iter->count, iter->nr_segs);
		}
		@@ -932,7 +961,6 @@ fail.
		iter->type, iter->iov_offset, iter->count, iter->nr_segs);
		pr_debug(" F nr_pages=0x%04x, size=0x%08x, offset==0x%04x", nr_pages, size, offset); // F nr_pages=0x009f, size=0x0009f000, offset==0x0000
		}

		return 0;
		}

		@@ -963,6 +991,48 @@ int bio_iov_iter_get_pages(struct bio bio, struct iov_iter iter)
		}
		EXPORT_SYMBOL_GPL(bio_iov_iter_get_pages);

		int bio_iov_iter_get_pages_elphel(struct bio bio, struct iov_iter iter)
		{
		int indx, n; // page index
		// size_t offset;
		// ssize_t size;
		size_t len;
		struct bio_vec *bv = bio->bi_io_vec + bio->bi_vcnt; // bio->bi_vcnt should be 0
		const struct iovec *iov = iter->iov; // only use first in array - supposed to be a single-element
		unsigned long paddr = 0; // physical address where to write
		unsigned long addr;
		unsigned long vaddr = (unsigned long) iov->iov_base;
		struct vm_area_struct *vm_area;
		if (iter->nr_segs != 1){
		pr_debug("nr_segs = %d != 1", iter->nr_segs);
		return -EINVAL;
		}

		// pr_debug("sizeof(struct vm_area_struct)=%d", sizeof(struct vm_area_struct));
		for (vm_area = current->mm->mmap; vm_area; vm_area = vm_area-> vm_next) {
		if ((vaddr >= vm_area->vm_start) && (vaddr < vm_area->vm_end)) {
		pr_debug("physical address = 0x%08lx", (unsigned long) vm_area->vm_private_data);
		print_hex_dump_bytes("", DUMP_PREFIX_OFFSET, vm_area, sizeof(struct vm_area_struct));
		paddr = (unsigned long) vm_area->vm_private_data;
		if (paddr) {
		paddr += vaddr - vm_area->vm_start; // added offset within buffer
		}
		break;
		}
		}
		if (!paddr){
		pr_debug("mmap-ed physical address is not specified in vm_area->vm_private_data, aborting");
		return -EINVAL;
		}
		pr_debug("virtual address = 0x%08lx, physical address = 0x%08lx", vaddr, paddr);

		bv[0].bv_page = NULL; // Make sure it will not be tried to release
		bv[0].bv_len = iov->iov_len;
		bv[0].bv_offset = 0;



		}

		static void submit_bio_wait_endio(struct bio *bio)
		{

src/block/blk-core.c

+3 −2

Original line number	Diff line number	Diff line
		@@ -2466,7 +2466,8 @@ blk_qc_t generic_make_request(struct bio *bio)
		}
		}
		if (bio->bi_disk && (bio->bi_disk->major == 8)) { // sda, sda*
		pr_debug(" { bio->bi_flags=0x%04x, bio->bi_phys_segments=0x%08x, enter_succeeded=%d",bio->bi_flags, bio->bi_phys_segments, enter_succeeded);
		pr_debug(" { state before blk_mq_make_request(): bio->bi_flags=0x%04x, bio->bi_phys_segments=0x%08x, enter_succeeded=%d",bio->bi_flags, bio->bi_phys_segments, enter_succeeded);
		print_hex_dump_bytes("", DUMP_PREFIX_OFFSET, bio, sizeof(struct bio));
		}
		if (enter_succeeded) {
		struct bio_list lower, same;
		@@ -2474,7 +2475,7 @@ blk_qc_t generic_make_request(struct bio *bio)
		/* Create a fresh bio_list for all subordinate requests */
		bio_list_on_stack[1] = bio_list_on_stack[0];
		bio_list_init(&bio_list_on_stack[0]);
		ret = q->make_request_fn(q, bio);
		ret = q->make_request_fn(q, bio); // in this case -> blk_mq_make_request()
		if (bio->bi_disk && (bio->bi_disk->major == 8)) { // sda, sda*
		pr_debug("q->make_request_fn(q, bio) -> %u", ret);
		}

src/block/blk-merge.c

+1 −0

Original line number	Diff line number	Diff line
		@@ -238,6 +238,7 @@ void blk_queue_split(struct request_queue q, struct bio *bio)
		split = blk_bio_write_same_split(q, *bio, &q->bio_split, &nsegs);
		break;
		default:
		// *** INCREASE q->limits.max_segment_size, it is set from BLK_MAX_SEGMENT_SIZE = 65536 (include/linux/blkdev.h:1429:) Elphel 06/14/2023
		split = blk_bio_segment_split(q, *bio, &q->bio_split, &nsegs);
		break;
		}

src/block/blk-mq.c

+2 −2

Original line number	Diff line number	Diff line
		@@ -1830,8 +1830,8 @@ static blk_qc_t blk_mq_make_request(struct request_queue q, struct bio bio)
		if (bio->bi_disk && (bio->bi_disk->major == 8)) { // sda, sda*
		pr_debug(" 1 bio->bi_flags=0x%04x, bio->bi_phys_segments=0x%08x",bio->bi_flags, bio->bi_phys_segments); // not yet set
		}
		// *** INCREASE q->limits.max_segment_size, it is set from BLK_MAX_SEGMENT_SIZE = 65536 (include/linux/blkdev.h:1429:) Elphel 06/14/2023
		blk_queue_split(q, &bio);

		if (bio->bi_disk && (bio->bi_disk->major == 8)) { // sda, sda* already split - 0xa segments (16 pages each)
		pr_debug(" 2 bio->bi_flags=0x%04x, bio->bi_phys_segments=0x%08x",bio->bi_flags, bio->bi_phys_segments); // already set
		}
		@@ -2592,7 +2592,7 @@ struct request_queue blk_mq_init_allocated_queue(struct blk_mq_tag_set set,
		struct request_queue *q)
		{
		// if (bio->bi_disk && (bio->bi_disk->major == 8)) { // sda, sda*
		pr_debug(" - ");
		pr_debug(" - "); // never
		// }
		/* mark the queue as mq asap */
		q->mq_ops = set->ops;

src/fs/block_dev.c

+11 −3

Original line number	Diff line number	Diff line
		@@ -190,7 +190,7 @@ __blkdev_direct_IO_simple(struct kiocb iocb, struct iov_iter iter,
		{
		struct file *file = iocb->ki_filp;
		struct block_device *bdev = I_BDEV(bdev_file_inode(file));
		struct bio_vec inline_vecs[DIO_INLINE_BIO_VECS], vecs, bvec;
		struct bio_vec inline_vecs[DIO_INLINE_BIO_VECS], vecs, bvec; // Elphel: may reduce it again
		loff_t pos = iocb->ki_pos;
		bool should_dirty = false;
		struct bio bio;
		@@ -222,7 +222,8 @@ __blkdev_direct_IO_simple(struct kiocb iocb, struct iov_iter iter,
		bio.bi_private = current;
		bio.bi_end_io = blkdev_bio_end_io_simple;
		bio.bi_ioprio = iocb->ki_ioprio;
		ret = bio_iov_iter_get_pages(&bio, iter);

		ret = bio_iov_iter_get_pages(&bio, iter); // normally returns 0
		pr_debug("ret = %d, bio.bi_vcnt=%d", ret, bio.bi_vcnt); // -14
		if (unlikely(ret))
		goto out;
		@@ -236,6 +237,7 @@ __blkdev_direct_IO_simple(struct kiocb iocb, struct iov_iter iter,
		bio.bi_opf = dio_bio_write_op(iocb);
		task_io_account_write(ret);
		}
		pr_debug("bio:");
		print_hex_dump_bytes("", DUMP_PREFIX_OFFSET, &bio, sizeof(struct bio));
		// pr_debug("bi_disk=0x%p, bi_sector=%llu, bi_size=%u, bi_private=0x%p, bi_vcnt=%u, bi_max_vecs=%u, bi_io_vec=0x%p, bi_pool=0x%p",
		// bio.bi_disk, bio.bi_iter.bi_sector, bio.bi_iter.bi_size, bio.bi_private, bio.bi_vcnt, bio.bi_max_vecs, bio.bi_io_vec, bio.bi_pool);
		@@ -250,8 +252,14 @@ __blkdev_direct_IO_simple(struct kiocb iocb, struct iov_iter iter,
		}
		}
		pr_debug("ARCH_PFN_OFFSET =0x%08lx, PHYS_PFN_OFFSET=0x%08lx, mem_map=0x%08x, bi_phys_segments=0x%08x", \
		ARCH_PFN_OFFSET, PHYS_PFN_OFFSET, (int) mem_map, bio.bi_phys_segments);
		ARCH_PFN_OFFSET, PHYS_PFN_OFFSET, (int) mem_map, bio.bi_phys_segments); // bi_phys_segments==0




		qc = submit_bio(&bio);
		pr_debug("After qc = submit_bio(&bio), bio:");
		print_hex_dump_bytes("", DUMP_PREFIX_OFFSET, &bio, sizeof(struct bio));
		for (;;) {
		set_current_state(TASK_UNINTERRUPTIBLE);
		if (!READ_ONCE(bio.bi_private))

Admin message