From 26b95c15f255904694ce8224b6f21f0b931a9cff Mon Sep 17 00:00:00 2001 From: Augustin Cavalier Date: Fri, 22 Nov 2019 22:09:48 -0500 Subject: [PATCH] freebsd_network: Completely overhaul bus_dma implementation. The old implementation was based on an ancient copy of the FreeBSD busdma code for x86, and did not make a bunch of assumptions that we make basically everywhere else (for instance, that we can request arbitrarily-aligned contiguous physical memory from the VM.) As a consequence, it had a significant amount of code devoted to bounce pages, which are just a waste of resources on x86, and for that matter, probably any other architecture Haiku will ever be ported to. (Even if we do need to run on some system where only a small portion of system memory can be accessed by devices, likely we would reserve that memory for just this occasion anyway.) I was initially under the impression that the bounce-pages code never turned on, but apparently due to the "alignment" check (and also the "Maxmem" check, which was to defined to 32KB...?!) it does indeed activate on a variety of systems, and maybe (in the case of drivers that do not call sync() properly) even is the cause of some of our ported driver breakage. The new implementation is pretty much optimized for Haiku, and shares almost no code or structure with the old one (save for a few functions that really only have one proper implementation.) Tested with ipro1000 and rtl81xx. Regressions are more than possible, so please don't hesitate to file bugs if your network driver now fails to come up (or you get KDLs.) --- src/libs/compat/freebsd_network/Jamfile | 4 +- src/libs/compat/freebsd_network/bus_dma.cpp | 380 ++++++ .../freebsd_network/compat/sys/bus_dma.h | 8 - .../compat/freebsd_network/compat/sys/param.h | 3 - src/libs/compat/freebsd_network/device.h | 4 - src/libs/compat/freebsd_network/driver.c | 4 - .../compat/freebsd_network/fbsd_busdma_x86.c | 1195 ----------------- 7 files changed, 382 insertions(+), 1216 deletions(-) create mode 100644 src/libs/compat/freebsd_network/bus_dma.cpp delete mode 100644 src/libs/compat/freebsd_network/fbsd_busdma_x86.c diff --git a/src/libs/compat/freebsd_network/Jamfile b/src/libs/compat/freebsd_network/Jamfile index b55eebfdcb..74b757c41a 100644 --- a/src/libs/compat/freebsd_network/Jamfile +++ b/src/libs/compat/freebsd_network/Jamfile @@ -10,10 +10,11 @@ UseHeaders $(HAIKU_PRIVATE_KERNEL_HEADERS) : true ; Includes [ FGristFiles kernel_c++_structs.h ] : kernel_c++_struct_sizes.h ; -SubDirCcFlags [ FDefines _KERNEL=1 _XOPEN_SOURCE ] ; +SubDirCcFlags [ FDefines _KERNEL=1 ] ; KernelStaticLibrary libfreebsd_network.a : bus.cpp + bus_dma.cpp callout.cpp clock.c condvar.c @@ -22,7 +23,6 @@ KernelStaticLibrary libfreebsd_network.a : device_hooks.c driver.c eventhandler.c - fbsd_busdma_x86.c fbsd_ether.c fbsd_if_media.c fbsd_kern_mbuf.c diff --git a/src/libs/compat/freebsd_network/bus_dma.cpp b/src/libs/compat/freebsd_network/bus_dma.cpp new file mode 100644 index 0000000000..6b588fadf5 --- /dev/null +++ b/src/libs/compat/freebsd_network/bus_dma.cpp @@ -0,0 +1,380 @@ +/* + * Copyright 2019, Haiku, Inc. All rights reserved. + * Distributed under the terms of the MIT License. + * + * Authors: + * Augustin Cavalier + */ + +extern "C" { +#include +#include +#include +#include +#include + +#include +} + + +// #pragma mark - structures + + +struct bus_dma_tag { + bus_dma_tag_t parent; + phys_size_t alignment; + phys_addr_t boundary; + phys_addr_t lowaddr; + phys_addr_t highaddr; + bus_dma_filter_t* filter; + void* filterarg; + phys_size_t maxsize; + uint32 nsegments; + phys_size_t maxsegsz; + int32 ref_count; + bus_dma_segment_t* segments; +}; + + +// #pragma mark - functions + + +void +busdma_lock_mutex(void* arg, bus_dma_lock_op_t op) +{ + struct mtx* dmtx = (struct mtx*)arg; + switch (op) { + case BUS_DMA_LOCK: + mtx_lock(dmtx); + break; + case BUS_DMA_UNLOCK: + mtx_unlock(dmtx); + break; + default: + panic("busdma_lock_mutex: unknown operation 0x%x", op); + } +} + + +int +bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment, bus_size_t boundary, + bus_addr_t lowaddr, bus_addr_t highaddr, bus_dma_filter_t* filter, + void* filterarg, bus_size_t maxsize, int nsegments, bus_size_t maxsegsz, + int flags, bus_dma_lock_t* lockfunc, void* lockfuncarg, bus_dma_tag_t* dmat) +{ + if (boundary != 0 && boundary < maxsegsz) + maxsegsz = boundary; + + *dmat = NULL; + + bus_dma_tag_t newtag = (bus_dma_tag_t)kernel_malloc(sizeof(*newtag), + M_DEVBUF, M_ZERO | M_NOWAIT); + if (newtag == NULL) + return ENOMEM; + + newtag->parent = parent; + newtag->alignment = alignment; + newtag->boundary = boundary; + newtag->lowaddr = lowaddr; + newtag->highaddr = highaddr; + newtag->filter = filter; + newtag->filterarg = filterarg; + newtag->maxsize = maxsize; + newtag->nsegments = nsegments; + newtag->maxsegsz = maxsegsz; + newtag->ref_count = 1; + + newtag->segments = (bus_dma_segment_t*)kernel_malloc( + sizeof(bus_dma_segment_t) * newtag->nsegments, M_DEVBUF, + M_NOWAIT); + if (newtag->segments == NULL) { + kernel_free(dmat, M_DEVBUF); + return ENOMEM; + } + + if (newtag->parent != NULL) { + atomic_add(&parent->ref_count, 1); + + newtag->lowaddr = max_c(parent->lowaddr, newtag->lowaddr); + newtag->highaddr = min_c(parent->highaddr, newtag->highaddr); + + if (newtag->boundary == 0) { + newtag->boundary = parent->boundary; + } else if (parent->boundary != 0) { + newtag->boundary = min_c(parent->boundary, newtag->boundary); + } + + if (newtag->filter == NULL) { + newtag->filter = parent->filter; + newtag->filterarg = parent->filterarg; + } + } + + if (newtag->filter != NULL) + panic("bus_dma_tag_create: error: filters not implemented!"); + + *dmat = newtag; + return 0; +} + + +int +bus_dma_tag_destroy(bus_dma_tag_t dmat) +{ + if (dmat == NULL) + return 0; + + while (dmat != NULL) { + bus_dma_tag_t parent; + + parent = dmat->parent; + atomic_add(&dmat->ref_count, -1); + if (dmat->ref_count == 0) { + kernel_free(dmat->segments, M_DEVBUF); + kernel_free(dmat, M_DEVBUF); + + // Last reference released, so release our reference on our parent. + dmat = parent; + } else + dmat = NULL; + } + return 0; +} + + +int +bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t* mapp) +{ + // We never bounce, so we do not need maps. + *mapp = NULL; + return 0; +} + + +int +bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map) +{ + // We never create maps, so we never need to destroy them. + if (map) + panic("map is not NULL!"); + return 0; +} + + +int +bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags, + bus_dmamap_t* mapp) +{ + int mflags; + if (flags & BUS_DMA_NOWAIT) + mflags = M_NOWAIT; + else + mflags = M_WAITOK; + + if (flags & BUS_DMA_ZERO) + mflags |= M_ZERO; + + // We never need to map/bounce. + *mapp = NULL; + + // FreeBSD uses standard malloc() for the case where maxsize <= PAGE_SIZE, + // however, our malloc() has no guarantees that the allocated memory will + // not be swapped out, which obviously is a requirement here. So we must + // always use kernel_contigmalloc(). + + // The range specified by lowaddr, highaddr is an *exclusion* range, + // not an inclusion range. So we want to at least start with the low end, + // if possible. (The most common exclusion range is 32-bit only, + // and ones other than that are very rare, so typically this will + // succeed.) + if (dmat->lowaddr > B_PAGE_SIZE) { + *vaddr = kernel_contigmalloc(dmat->maxsize, M_DEVBUF, mflags, + 0, dmat->lowaddr, + dmat->alignment ? dmat->alignment : 1ul, dmat->boundary); + if (*vaddr == NULL) + dprintf("bus_dmamem_alloc: failed to allocate with lowaddr " + "0x%" B_PRIxPHYSADDR "\n", dmat->lowaddr); + } + if (*vaddr == NULL && dmat->highaddr < BUS_SPACE_MAXADDR) { + *vaddr = kernel_contigmalloc(dmat->maxsize, M_DEVBUF, mflags, + dmat->highaddr, BUS_SPACE_MAXADDR, + dmat->alignment ? dmat->alignment : 1ul, dmat->boundary); + } + + if (*vaddr == NULL) { + dprintf("bus_dmamem_alloc: failed to allocate for tag (size %d, " + "low 0x%" B_PRIxPHYSADDR ", high 0x%" B_PRIxPHYSADDR ", " + "boundary 0x%" B_PRIxPHYSADDR ")\n", + (int)dmat->maxsize, dmat->lowaddr, dmat->highaddr, dmat->boundary); + return ENOMEM; + } else if (vtophys(*vaddr) & (dmat->alignment - 1)) { + dprintf("bus_dmamem_alloc: failed to align memory: wanted %#x, got %#x\n", + dmat->alignment, vtophys(vaddr)); + bus_dmamem_free(dmat, *vaddr, *mapp); + return ENOMEM; + } + return 0; +} + + +void +bus_dmamem_free(bus_dma_tag_t dmat, void* vaddr, bus_dmamap_t map) +{ + // We never bounce, so map should be NULL. + if (map != NULL) + panic("bus_dmamem_free: map is not NULL!"); + + kernel_contigfree(vaddr, dmat->maxsize, M_DEVBUF); +} + + +static int +_bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dmamap_t /* map */, void* buf, + bus_size_t buflen, int flags, bus_addr_t* lastaddrp, bus_dma_segment_t* segs, + int& seg, bool first) +{ + vm_offset_t virtual_addr = (vm_offset_t)buf; + bus_addr_t last_phys_addr = *lastaddrp; + const bus_addr_t boundary_mask = ~(dmat->boundary - 1); + + while (buflen > 0) { + const bus_addr_t phys_addr = pmap_kextract(virtual_addr); + + bus_size_t segment_size = B_PAGE_SIZE - (phys_addr & (B_PAGE_SIZE - 1)); + if (segment_size > buflen) + segment_size = buflen; + + if (dmat->boundary > 0) { + // Make sure we don't cross a boundary. + bus_addr_t boundary_addr = (phys_addr + dmat->boundary) & boundary_mask; + if (segment_size > (boundary_addr - phys_addr)) + segment_size = (boundary_addr - phys_addr); + } + + // Insert chunk into a segment. + if (first) { + segs[seg].ds_addr = phys_addr; + segs[seg].ds_len = segment_size; + first = false; + } else { + // If possible, coalesce into the previous segment. + if (phys_addr == last_phys_addr + && (segs[seg].ds_len + segment_size) <= dmat->maxsegsz + && (dmat->boundary == 0 + || (segs[seg].ds_addr & boundary_mask) + == (phys_addr & boundary_mask))) { + segs[seg].ds_len += segment_size; + } else { + if (++seg >= dmat->nsegments) + break; + segs[seg].ds_addr = phys_addr; + segs[seg].ds_len = segment_size; + } + } + + last_phys_addr = phys_addr + segment_size; + virtual_addr += segment_size; + buflen -= segment_size; + } + + *lastaddrp = last_phys_addr; + return (buflen != 0 ? EFBIG : 0); +} + + +int +bus_dmamap_load(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf, + bus_size_t buflen, bus_dmamap_callback_t *callback, + void *callback_arg, int flags) +{ + bus_addr_t lastaddr = 0; + int error, nsegs = 0; + + error = _bus_dmamap_load_buffer(dmat, map, buf, buflen, flags, + &lastaddr, dmat->segments, nsegs, true); + + if (error) + (*callback)(callback_arg, dmat->segments, 0, error); + else + (*callback)(callback_arg, dmat->segments, nsegs + 1, 0); + + // ENOMEM is returned; all other errors are only sent to the callback. + if (error == ENOMEM) + return error; + return 0; +} + + +int +bus_dmamap_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map, struct mbuf* mb, + bus_dmamap_callback2_t* callback, void* callback_arg, int flags) +{ + M_ASSERTPKTHDR(mb); + + int nsegs = 0, error = 0; + if (mb->m_pkthdr.len <= dmat->maxsize) { + bool first = true; + bus_addr_t lastaddr = 0; + for (struct mbuf* m = mb; m != NULL && error == 0; m = m->m_next) { + if (m->m_len <= 0) + continue; + + error = _bus_dmamap_load_buffer(dmat, map, m->m_data, m->m_len, + flags, &lastaddr, dmat->segments, nsegs, first); + first = false; + } + } else { + error = EINVAL; + } + + if (error) { + (*callback)(callback_arg, dmat->segments, 0, 0, error); + } else { + (*callback)(callback_arg, dmat->segments, nsegs + 1, mb->m_pkthdr.len, + error); + } + return error; +} + + +int +bus_dmamap_load_mbuf_sg(bus_dma_tag_t dmat, bus_dmamap_t map, struct mbuf* mb, + bus_dma_segment_t* segs, int* nsegs, int flags) +{ + M_ASSERTPKTHDR(mb); + + *nsegs = 0; + int error = 0; + if (mb->m_pkthdr.len <= dmat->maxsize) { + bool first = true; + bus_addr_t lastaddr = 0; + + for (struct mbuf* m = mb; m != NULL && error == 0; m = m->m_next) { + if (m->m_len <= 0) + continue; + + error = _bus_dmamap_load_buffer(dmat, map, m->m_data, m->m_len, + flags, &lastaddr, segs, *nsegs, first); + first = false; + } + } else { + error = EINVAL; + } + + ++*nsegs; + return error; +} + + +void +_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map) +{ + // We never allocate bounce pages; nothing to do. +} + + +void +_bus_dmamap_sync(bus_dma_tag_t, bus_dmamap_t, bus_dmasync_op_t) +{ + // We never bounce; nothing to do. +} diff --git a/src/libs/compat/freebsd_network/compat/sys/bus_dma.h b/src/libs/compat/freebsd_network/compat/sys/bus_dma.h index 362019f783..d24753ac23 100644 --- a/src/libs/compat/freebsd_network/compat/sys/bus_dma.h +++ b/src/libs/compat/freebsd_network/compat/sys/bus_dma.h @@ -184,13 +184,6 @@ typedef void bus_dmamap_callback_t(void *, bus_dma_segment_t *, int, int); */ typedef void bus_dmamap_callback2_t(void *, bus_dma_segment_t *, int, bus_size_t, int); -/* - * XXX sparc64 uses the same interface, but a much different implementation. - * for the sparc64 arch contains the equivalent - * declarations. - */ -#if !defined(__sparc64__) - /* * Allocate a handle for mapping from kva/uva/physical * address space into bus device space. @@ -266,6 +259,5 @@ void _bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map); _bus_dmamap_unload(dmat, dmamap); \ } while (0) -#endif /* __sparc64__ */ #endif /* _BUS_DMA_H_ */ diff --git a/src/libs/compat/freebsd_network/compat/sys/param.h b/src/libs/compat/freebsd_network/compat/sys/param.h index 9d7ea717f2..bf7770beff 100644 --- a/src/libs/compat/freebsd_network/compat/sys/param.h +++ b/src/libs/compat/freebsd_network/compat/sys/param.h @@ -30,9 +30,6 @@ #define ptoa(x) ((unsigned long)((x) << PAGE_SHIFT)) #define atop(x) ((unsigned long)((x) >> PAGE_SHIFT)) -/* MAJOR FIXME */ -#define Maxmem (32768) - #ifndef MSIZE #define MSIZE 256 #endif diff --git a/src/libs/compat/freebsd_network/device.h b/src/libs/compat/freebsd_network/device.h index fbf04ec347..d1b169df66 100644 --- a/src/libs/compat/freebsd_network/device.h +++ b/src/libs/compat/freebsd_network/device.h @@ -76,10 +76,6 @@ void uninit_callout(void); device_t find_root_device(int); -/* busdma_machdep.c */ -void init_bounce_pages(void); -void uninit_bounce_pages(void); - void driver_printf(const char *format, ...) __attribute__ ((format (__printf__, 1, 2))); int driver_vprintf(const char *format, va_list vl); diff --git a/src/libs/compat/freebsd_network/driver.c b/src/libs/compat/freebsd_network/driver.c index d4f4e081b8..10f1bcf32e 100644 --- a/src/libs/compat/freebsd_network/driver.c +++ b/src/libs/compat/freebsd_network/driver.c @@ -193,8 +193,6 @@ _fbsd_init_drivers(driver_t *drivers[]) if (status < B_OK) goto err4; - init_bounce_pages(); - if (HAIKU_DRIVER_REQUIRES(FBSD_TASKQUEUES)) { status = init_taskqueues(); if (status < B_OK) @@ -244,7 +242,6 @@ err6: if (HAIKU_DRIVER_REQUIRES(FBSD_TASKQUEUES)) uninit_taskqueues(); err5: - uninit_bounce_pages(); uninit_callout(); err4: uninit_mbufs(); @@ -281,7 +278,6 @@ _fbsd_uninit_drivers(driver_t *drivers[]) uninit_sysinit(); if (HAIKU_DRIVER_REQUIRES(FBSD_TASKQUEUES)) uninit_taskqueues(); - uninit_bounce_pages(); uninit_callout(); uninit_mbufs(); uninit_mutexes(); diff --git a/src/libs/compat/freebsd_network/fbsd_busdma_x86.c b/src/libs/compat/freebsd_network/fbsd_busdma_x86.c deleted file mode 100644 index 25fc6d5e45..0000000000 --- a/src/libs/compat/freebsd_network/fbsd_busdma_x86.c +++ /dev/null @@ -1,1195 +0,0 @@ -/*- - * Copyright (c) 1997, 1998 Justin T. Gibbs. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions, and the following disclaimer, - * without modification, immediately at the beginning of the file. - * 2. The name of the author may not be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR - * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ -#include -__FBSDID("$FreeBSD: src/sys/i386/i386/busdma_machdep.c,v 1.74.2.4 2006/10/21 16:26:53 hrs Exp $"); - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include -#include - -#ifdef __x86_64__ -# define MAX_BPAGES 8192 -#else -# define MAX_BPAGES 512 -#endif - -/* -hugo */ -#define malloc(a, b, c) kernel_malloc(a, b, c) -#define free(a, b) kernel_free(a, b) -#define contigmalloc(a, b, c, d, e, f, g) kernel_contigmalloc(a, b, c, d, e, f, g) -#define contigfree(a, b, c) kernel_contigfree(a, b, c) -void busdma_swi(void); -void init_bounce_pages(void); -void uninit_bounce_pages(void); -/* */ - -struct bounce_zone; - -struct bus_dma_tag { - bus_dma_tag_t parent; - bus_size_t alignment; - bus_addr_t boundary; - bus_addr_t lowaddr; - bus_addr_t highaddr; - bus_dma_filter_t *filter; - void *filterarg; - bus_size_t maxsize; - u_int nsegments; - bus_size_t maxsegsz; - int flags; - int ref_count; - int map_count; - bus_dma_lock_t *lockfunc; - void *lockfuncarg; - bus_dma_segment_t *segments; - struct bounce_zone *bounce_zone; -}; - -struct bounce_page { - vm_offset_t vaddr; /* kva of bounce buffer */ - bus_addr_t busaddr; /* Physical address */ - vm_offset_t datavaddr; /* kva of client data */ - bus_size_t datacount; /* client data count */ - STAILQ_ENTRY(bounce_page) links; -}; - -int busdma_swi_pending; - -struct bounce_zone { - STAILQ_ENTRY(bounce_zone) links; - STAILQ_HEAD(bp_list, bounce_page) bounce_page_list; - int total_bpages; - int free_bpages; - int reserved_bpages; - int active_bpages; - int total_bounced; - int total_deferred; - bus_size_t alignment; - bus_addr_t boundary; - bus_addr_t lowaddr; - char zoneid[8]; - char lowaddrid[20]; - struct sysctl_ctx_list sysctl_tree; - struct sysctl_oid *sysctl_tree_top; -}; - -static struct mtx bounce_lock; -static int total_bpages; -static int busdma_zonecount; -static STAILQ_HEAD(, bounce_zone) bounce_zone_list; - -SYSCTL_NODE(_hw, OID_AUTO, busdma, CTLFLAG_RD, 0, "Busdma parameters"); -SYSCTL_INT(_hw_busdma, OID_AUTO, total_bpages, CTLFLAG_RD, &total_bpages, 0, - "Total bounce pages"); - -struct bus_dmamap { - struct bp_list bpages; - int pagesneeded; - int pagesreserved; - bus_dma_tag_t dmat; - void *buf; /* unmapped buffer pointer */ - bus_size_t buflen; /* unmapped buffer length */ - bus_dmamap_callback_t *callback; - void *callback_arg; - STAILQ_ENTRY(bus_dmamap) links; -}; - -static STAILQ_HEAD(, bus_dmamap) bounce_map_waitinglist; -static STAILQ_HEAD(, bus_dmamap) bounce_map_callbacklist; -static struct bus_dmamap nobounce_dmamap; - -static int alloc_bounce_zone(bus_dma_tag_t dmat); -static int alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages); -static int reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, - int commit); -static bus_addr_t add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, - vm_offset_t vaddr, bus_size_t size); -static void free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage); -static __inline int run_filter(bus_dma_tag_t dmat, bus_addr_t paddr); - -/* - * Return true if a match is made. - * - * To find a match walk the chain of bus_dma_tag_t's looking for 'paddr'. - * - * If paddr is within the bounds of the dma tag then call the filter callback - * to check for a match, if there is no filter callback then assume a match. - */ -static __inline -int -run_filter(bus_dma_tag_t dmat, bus_addr_t paddr) -{ - int retval; - - retval = 0; - - do { - if (((paddr > dmat->lowaddr && paddr <= dmat->highaddr) - || ((paddr & (dmat->alignment - 1)) != 0)) - && (dmat->filter == NULL - || (*dmat->filter)(dmat->filterarg, paddr) != 0)) - retval = 1; - - dmat = dmat->parent; - } while (retval == 0 && dmat != NULL); - return (retval); -} - -/* - * Convenience function for manipulating driver locks from busdma (during - * busdma_swi, for example). Drivers that don't provide their own locks - * should specify &Giant to dmat->lockfuncarg. Drivers that use their own - * non-mutex locking scheme don't have to use this at all. - */ -void -busdma_lock_mutex(void *arg, bus_dma_lock_op_t op) -{ - struct mtx *dmtx; - - dmtx = (struct mtx *)arg; - switch (op) { - case BUS_DMA_LOCK: - mtx_lock(dmtx); - break; - case BUS_DMA_UNLOCK: - mtx_unlock(dmtx); - break; - default: - panic("Unknown operation 0x%x for busdma_lock_mutex!", op); - } -} - -/* - * dflt_lock should never get called. It gets put into the dma tag when - * lockfunc == NULL, which is only valid if the maps that are associated - * with the tag are meant to never be defered. - * XXX Should have a way to identify which driver is responsible here. - */ -static -void -dflt_lock(void *arg, bus_dma_lock_op_t op) -{ - panic("driver error: busdma dflt_lock called"); -} - -#define BUS_DMA_COULD_BOUNCE BUS_DMA_BUS3 -#define BUS_DMA_MIN_ALLOC_COMP BUS_DMA_BUS4 - -/* - * Allocate a device specific dma_tag. - */ -int -bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment, - bus_size_t boundary, bus_addr_t lowaddr, - bus_addr_t highaddr, bus_dma_filter_t *filter, - void *filterarg, bus_size_t maxsize, int nsegments, - bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc, - void *lockfuncarg, bus_dma_tag_t *dmat) -{ - bus_dma_tag_t newtag; - int error = 0; - - /* Basic sanity checking */ - if (boundary != 0 && boundary < maxsegsz) - maxsegsz = boundary; - - /* Return a NULL tag on failure */ - *dmat = NULL; - - newtag = (bus_dma_tag_t)malloc(sizeof(*newtag), M_DEVBUF, - M_ZERO | M_NOWAIT); - if (newtag == NULL) { - CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", - __func__, newtag, 0, error); - return (ENOMEM); - } - - newtag->parent = parent; - newtag->alignment = alignment; - newtag->boundary = boundary; - newtag->lowaddr = trunc_page((vm_paddr_t)lowaddr) + (PAGE_SIZE - 1); - newtag->highaddr = trunc_page((vm_paddr_t)highaddr) + - (PAGE_SIZE - 1); - newtag->filter = filter; - newtag->filterarg = filterarg; - newtag->maxsize = maxsize; - newtag->nsegments = nsegments; - newtag->maxsegsz = maxsegsz; - newtag->flags = flags; - newtag->ref_count = 1; /* Count ourself */ - newtag->map_count = 0; - if (lockfunc != NULL) { - newtag->lockfunc = lockfunc; - newtag->lockfuncarg = lockfuncarg; - } else { - newtag->lockfunc = dflt_lock; - newtag->lockfuncarg = NULL; - } - newtag->segments = NULL; - - /* Take into account any restrictions imposed by our parent tag */ - if (parent != NULL) { - newtag->lowaddr = MIN(parent->lowaddr, newtag->lowaddr); - newtag->highaddr = MAX(parent->highaddr, newtag->highaddr); - if (newtag->boundary == 0) - newtag->boundary = parent->boundary; - else if (parent->boundary != 0) - newtag->boundary = MIN(parent->boundary, - newtag->boundary); - if (newtag->filter == NULL) { - /* - * Short circuit looking at our parent directly - * since we have encapsulated all of its information - */ - newtag->filter = parent->filter; - newtag->filterarg = parent->filterarg; - newtag->parent = parent->parent; - } - if (newtag->parent != NULL) - atomic_add_int(&parent->ref_count, 1); - } - - if (newtag->lowaddr < ptoa((vm_paddr_t)Maxmem) - || newtag->alignment > 1) - newtag->flags |= BUS_DMA_COULD_BOUNCE; - - if (((newtag->flags & BUS_DMA_COULD_BOUNCE) != 0) && - (flags & BUS_DMA_ALLOCNOW) != 0) { - struct bounce_zone *bz; - - /* Must bounce */ - - if ((error = alloc_bounce_zone(newtag)) != 0) { - free(newtag, M_DEVBUF); - return (error); - } - bz = newtag->bounce_zone; - - if (ptoa(bz->total_bpages) < maxsize) { - int pages; - - pages = atop(maxsize) - bz->total_bpages; - - /* Add pages to our bounce pool */ - if (alloc_bounce_pages(newtag, pages) < pages) - error = ENOMEM; - } - /* Performed initial allocation */ - newtag->flags |= BUS_DMA_MIN_ALLOC_COMP; - } - - if (error != 0) { - free(newtag, M_DEVBUF); - } else { - *dmat = newtag; - } - CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", - __func__, newtag, (newtag != NULL ? newtag->flags : 0), error); - return (error); -} - -int -bus_dma_tag_destroy(bus_dma_tag_t dmat) -{ - bus_dma_tag_t dmat_copy; - int error; - - error = 0; - dmat_copy = dmat; - - if (dmat != NULL) { - - if (dmat->map_count != 0) { - error = EBUSY; - goto out; - } - - while (dmat != NULL) { - bus_dma_tag_t parent; - - parent = dmat->parent; - atomic_subtract_int(&dmat->ref_count, 1); - if (dmat->ref_count == 0) { - if (dmat->segments != NULL) - free(dmat->segments, M_DEVBUF); - free(dmat, M_DEVBUF); - /* - * Last reference count, so - * release our reference - * count on our parent. - */ - dmat = parent; - } else - dmat = NULL; - } - } -out: - CTR3(KTR_BUSDMA, "%s tag %p error %d", __func__, dmat_copy, error); - return (error); -} - -/* - * Allocate a handle for mapping from kva/uva/physical - * address space into bus device space. - */ -int -bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp) -{ - int error; - - error = 0; - - if (dmat->segments == NULL) { - dmat->segments = (bus_dma_segment_t *)malloc( - sizeof(bus_dma_segment_t) * dmat->nsegments, M_DEVBUF, - M_NOWAIT); - if (dmat->segments == NULL) { - CTR3(KTR_BUSDMA, "%s: tag %p error %d", - __func__, dmat, ENOMEM); - return (ENOMEM); - } - } - - /* - * Bouncing might be required if the driver asks for an active - * exclusion region, a data alignment that is stricter than 1, and/or - * an active address boundary. - */ - if (dmat->flags & BUS_DMA_COULD_BOUNCE) { - - /* Must bounce */ - struct bounce_zone *bz; - int maxpages; - - if (dmat->bounce_zone == NULL) { - if ((error = alloc_bounce_zone(dmat)) != 0) - return (error); - } - bz = dmat->bounce_zone; - - *mapp = (bus_dmamap_t)malloc(sizeof(**mapp), M_DEVBUF, - M_NOWAIT | M_ZERO); - if (*mapp == NULL) { - CTR3(KTR_BUSDMA, "%s: tag %p error %d", - __func__, dmat, ENOMEM); - return (ENOMEM); - } - - /* Initialize the new map */ - STAILQ_INIT(&((*mapp)->bpages)); - - /* - * Attempt to add pages to our pool on a per-instance - * basis up to a sane limit. - */ - if (dmat->alignment > 1) - maxpages = MAX_BPAGES; - else - maxpages = MIN(MAX_BPAGES, Maxmem -atop(dmat->lowaddr)); - if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0 - || (dmat->map_count > 0 && bz->total_bpages < maxpages)) { - int pages; - - pages = MAX(atop(dmat->maxsize), 1); - pages = MIN(maxpages - bz->total_bpages, pages); - pages = MAX(pages, 1); - if (alloc_bounce_pages(dmat, pages) < pages) - error = ENOMEM; - - if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0) { - if (error == 0) - dmat->flags |= BUS_DMA_MIN_ALLOC_COMP; - } else { - error = 0; - } - } - } else { - *mapp = NULL; - } - if (error == 0) - dmat->map_count++; - CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", - __func__, dmat, dmat->flags, error); - return (error); -} - -/* - * Destroy a handle for mapping from kva/uva/physical - * address space into bus device space. - */ -int -bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map) -{ - if (map != NULL && map != &nobounce_dmamap) { - if (STAILQ_FIRST(&map->bpages) != NULL) { - CTR3(KTR_BUSDMA, "%s: tag %p error %d", - __func__, dmat, EBUSY); - return (EBUSY); - } - free(map, M_DEVBUF); - } - dmat->map_count--; - CTR2(KTR_BUSDMA, "%s: tag %p error 0", __func__, dmat); - return (0); -} - -/* - * Allocate a piece of memory that can be efficiently mapped into - * bus device space based on the constraints lited in the dma tag. - * A dmamap to for use with dmamap_load is also allocated. - */ -int -bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags, - bus_dmamap_t *mapp) -{ - int mflags; - - if (flags & BUS_DMA_NOWAIT) - mflags = M_NOWAIT; - else - mflags = M_WAITOK; - if (flags & BUS_DMA_ZERO) - mflags |= M_ZERO; - - /* If we succeed, no mapping/bouncing will be required */ - *mapp = NULL; - - if (dmat->segments == NULL) { - dmat->segments = (bus_dma_segment_t *)malloc( - sizeof(bus_dma_segment_t) * dmat->nsegments, M_DEVBUF, - M_NOWAIT); - if (dmat->segments == NULL) { - CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", - __func__, dmat, dmat->flags, ENOMEM); - return (ENOMEM); - } - } - - /* - * XXX: - * (dmat->alignment < dmat->maxsize) is just a quick hack; the exact - * alignment guarantees of malloc need to be nailed down, and the - * code below should be rewritten to take that into account. - * - * In the meantime, we'll warn the user if malloc gets it wrong. - */ - if ((dmat->maxsize <= PAGE_SIZE) && - (dmat->alignment < dmat->maxsize) && - dmat->lowaddr >= ptoa((vm_paddr_t)Maxmem)) { - *vaddr = malloc(dmat->maxsize, M_DEVBUF, mflags); - } else { - /* - * XXX Use Contigmalloc until it is merged into this facility - * and handles multi-seg allocations. Nobody is doing - * multi-seg allocations yet though. - * XXX Certain AGP hardware does. - */ - *vaddr = contigmalloc(dmat->maxsize, M_DEVBUF, mflags, - 0ul, dmat->lowaddr, dmat->alignment? dmat->alignment : 1ul, - dmat->boundary); - } - if (*vaddr == NULL) { - CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", - __func__, dmat, dmat->flags, ENOMEM); - return (ENOMEM); - } else if (vtophys(*vaddr) & (dmat->alignment - 1)) { - printf("bus_dmamem_alloc failed to align memory: wanted %#x, got %#x\n", - dmat->alignment, vtophys(vaddr)); - } - CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", - __func__, dmat, dmat->flags, 0); - return (0); -} - -/* - * Free a piece of memory and it's allociated dmamap, that was allocated - * via bus_dmamem_alloc. Make the same choice for free/contigfree. - */ -void -bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map) -{ - /* - * dmamem does not need to be bounced, so the map should be - * NULL - */ - if (map != NULL) - panic("bus_dmamem_free: Invalid map freed\n"); - if ((dmat->maxsize <= PAGE_SIZE) && - (dmat->alignment < dmat->maxsize) && - dmat->lowaddr >= ptoa((vm_paddr_t)Maxmem)) - free(vaddr, M_DEVBUF); - else { - contigfree(vaddr, dmat->maxsize, M_DEVBUF); - } - CTR3(KTR_BUSDMA, "%s: tag %p flags 0x%x", __func__, dmat, dmat->flags); -} - -/* - * Utility function to load a linear buffer. lastaddrp holds state - * between invocations (for multiple-buffer loads). segp contains - * the starting segment on entrace, and the ending segment on exit. - * first indicates if this is the first invocation of this function. - */ -static __inline int -_bus_dmamap_load_buffer(bus_dma_tag_t dmat, - bus_dmamap_t map, - void *buf, bus_size_t buflen, - pmap_t pmap, - int flags, - bus_addr_t *lastaddrp, - bus_dma_segment_t *segs, - int *segp, - int first) -{ - bus_size_t sgsize; - bus_addr_t curaddr, lastaddr, baddr, bmask; - vm_offset_t vaddr; - bus_addr_t paddr; - int needbounce = 0; - int seg; - - if (map == NULL) - map = &nobounce_dmamap; - - if ((map != &nobounce_dmamap && map->pagesneeded == 0) - && ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0)) { - vm_offset_t vendaddr; - - CTR4(KTR_BUSDMA, "lowaddr= %d Maxmem= %d, boundary= %d, " - "alignment= %d", dmat->lowaddr, ptoa((vm_paddr_t)Maxmem), - dmat->boundary, dmat->alignment); - CTR3(KTR_BUSDMA, "map= %p, nobouncemap= %p, pagesneeded= %d", - map, &nobounce_dmamap, map->pagesneeded); - /* - * Count the number of bounce pages - * needed in order to complete this transfer - */ - vaddr = trunc_page((vm_offset_t)buf); - vendaddr = (vm_offset_t)buf + buflen; - - while (vaddr < vendaddr) { - paddr = pmap_kextract(vaddr); - if (run_filter(dmat, paddr) != 0) { - needbounce = 1; - map->pagesneeded++; - } - vaddr += PAGE_SIZE; - } - CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded); - } - - /* Reserve Necessary Bounce Pages */ - if (map->pagesneeded != 0) { - mtx_lock(&bounce_lock); - if (flags & BUS_DMA_NOWAIT) { - if (reserve_bounce_pages(dmat, map, 0) != 0) { - mtx_unlock(&bounce_lock); - return (ENOMEM); - } - } else { - if (reserve_bounce_pages(dmat, map, 1) != 0) { - /* Queue us for resources */ - map->dmat = dmat; - map->buf = buf; - map->buflen = buflen; - STAILQ_INSERT_TAIL(&bounce_map_waitinglist, - map, links); - mtx_unlock(&bounce_lock); - return (EINPROGRESS); - } - } - mtx_unlock(&bounce_lock); - } - - vaddr = (vm_offset_t)buf; - lastaddr = *lastaddrp; - bmask = ~(dmat->boundary - 1); - - for (seg = *segp; buflen > 0 ; ) { - /* - * Get the physical address for this segment. - */ - if (pmap) - curaddr = (bus_addr_t)pmap_extract(pmap, vaddr); - else - curaddr = pmap_kextract(vaddr); - - /* - * Compute the segment size, and adjust counts. - */ - sgsize = PAGE_SIZE - ((u_long)curaddr & PAGE_MASK); - if (buflen < sgsize) - sgsize = buflen; - - /* - * Make sure we don't cross any boundaries. - */ - if (dmat->boundary > 0) { - baddr = (curaddr + dmat->boundary) & bmask; - if (sgsize > (baddr - curaddr)) - sgsize = (baddr - curaddr); - } - - if (map->pagesneeded != 0 && run_filter(dmat, curaddr)) - curaddr = add_bounce_page(dmat, map, vaddr, sgsize); - - /* - * Insert chunk into a segment, coalescing with - * previous segment if possible. - */ - if (first) { - segs[seg].ds_addr = curaddr; - segs[seg].ds_len = sgsize; - first = 0; - } else { - if (needbounce == 0 && curaddr == lastaddr && - (segs[seg].ds_len + sgsize) <= dmat->maxsegsz && - (dmat->boundary == 0 || - (segs[seg].ds_addr & bmask) == (curaddr & bmask))) - segs[seg].ds_len += sgsize; - else { - if (++seg >= dmat->nsegments) - break; - segs[seg].ds_addr = curaddr; - segs[seg].ds_len = sgsize; - } - } - - lastaddr = curaddr + sgsize; - vaddr += sgsize; - buflen -= sgsize; - } - - *segp = seg; - *lastaddrp = lastaddr; - - /* - * Did we fit? - */ - return (buflen != 0 ? EFBIG : 0); /* XXX better return value here? */ -} - -/* - * Map the buffer buf into bus space using the dmamap map. - */ -int -bus_dmamap_load(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf, - bus_size_t buflen, bus_dmamap_callback_t *callback, - void *callback_arg, int flags) -{ - bus_addr_t lastaddr = 0; - int error, nsegs = 0; - - if (map != NULL) { - flags |= BUS_DMA_WAITOK; - map->callback = callback; - map->callback_arg = callback_arg; - } - - error = _bus_dmamap_load_buffer(dmat, map, buf, buflen, NULL, flags, - &lastaddr, dmat->segments, &nsegs, 1); - - CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", - __func__, dmat, dmat->flags, error, nsegs + 1); - - if (error == EINPROGRESS) { - return (error); - } - - if (error) - (*callback)(callback_arg, dmat->segments, 0, error); - else - (*callback)(callback_arg, dmat->segments, nsegs + 1, 0); - - /* - * Return ENOMEM to the caller so that it can pass it up the stack. - * This error only happens when NOWAIT is set, so deferal is disabled. - */ - if (error == ENOMEM) - return (error); - - return (0); -} - -/* - * Like _bus_dmamap_load(), but for mbufs. - */ -int -bus_dmamap_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map, - struct mbuf *m0, - bus_dmamap_callback2_t *callback, void *callback_arg, - int flags) -{ - int nsegs, error; - - M_ASSERTPKTHDR(m0); - - flags |= BUS_DMA_NOWAIT; - nsegs = 0; - error = 0; - if (m0->m_pkthdr.len <= dmat->maxsize) { - int first = 1; - bus_addr_t lastaddr = 0; - struct mbuf *m; - - for (m = m0; m != NULL && error == 0; m = m->m_next) { - if (m->m_len > 0) { - error = _bus_dmamap_load_buffer(dmat, map, - m->m_data, m->m_len, - NULL, flags, &lastaddr, - dmat->segments, &nsegs, first); - first = 0; - } - } - } else { - error = EINVAL; - } - - if (error) { - /* force "no valid mappings" in callback */ - (*callback)(callback_arg, dmat->segments, 0, 0, error); - } else { - (*callback)(callback_arg, dmat->segments, - nsegs+1, m0->m_pkthdr.len, error); - } - CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", - __func__, dmat, dmat->flags, error, nsegs + 1); - return (error); -} - -int -bus_dmamap_load_mbuf_sg(bus_dma_tag_t dmat, bus_dmamap_t map, - struct mbuf *m0, bus_dma_segment_t *segs, int *nsegs, - int flags) -{ - int error; - - M_ASSERTPKTHDR(m0); - - flags |= BUS_DMA_NOWAIT; - *nsegs = 0; - error = 0; - if (m0->m_pkthdr.len <= dmat->maxsize) { - int first = 1; - bus_addr_t lastaddr = 0; - struct mbuf *m; - - for (m = m0; m != NULL && error == 0; m = m->m_next) { - if (m->m_len > 0) { - error = _bus_dmamap_load_buffer(dmat, map, - m->m_data, m->m_len, - NULL, flags, &lastaddr, - segs, nsegs, first); - first = 0; - } - } - } else { - error = EINVAL; - } - - /* XXX FIXME: Having to increment nsegs is really annoying */ - ++*nsegs; - CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", - __func__, dmat, dmat->flags, error, *nsegs); - return (error); -} - -#if 0 -/* - * Like _bus_dmamap_load(), but for uios. - */ -int -bus_dmamap_load_uio(bus_dma_tag_t dmat, bus_dmamap_t map, - struct uio *uio, - bus_dmamap_callback2_t *callback, void *callback_arg, - int flags) -{ - bus_addr_t lastaddr; - int nsegs, error, first, i; - bus_size_t resid; - struct iovec *iov; - pmap_t pmap; - - flags |= BUS_DMA_NOWAIT; - resid = uio->uio_resid; - iov = uio->uio_iov; - - if (uio->uio_segflg == UIO_USERSPACE) { - KASSERT(uio->uio_td != NULL, - ("bus_dmamap_load_uio: USERSPACE but no proc")); - pmap = vmspace_pmap(uio->uio_td->td_proc->p_vmspace); - } else - pmap = NULL; - - nsegs = 0; - error = 0; - first = 1; - for (i = 0; i < uio->uio_iovcnt && resid != 0 && !error; i++) { - /* - * Now at the first iovec to load. Load each iovec - * until we have exhausted the residual count. - */ - bus_size_t minlen = - resid < iov[i].iov_len ? resid : iov[i].iov_len; - caddr_t addr = (caddr_t) iov[i].iov_base; - - if (minlen > 0) { - error = _bus_dmamap_load_buffer(dmat, map, - addr, minlen, pmap, flags, &lastaddr, - dmat->segments, &nsegs, first); - first = 0; - - resid -= minlen; - } - } - - if (error) { - /* force "no valid mappings" in callback */ - (*callback)(callback_arg, dmat->segments, 0, 0, error); - } else { - (*callback)(callback_arg, dmat->segments, - nsegs+1, uio->uio_resid, error); - } - CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", - __func__, dmat, dmat->flags, error, nsegs + 1); - return (error); -} -#endif - -/* - * Release the mapping held by map. - */ -void -_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map) -{ - struct bounce_page *bpage; - - while ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) { - STAILQ_REMOVE_HEAD(&map->bpages, links); - free_bounce_page(dmat, bpage); - } -} - -void -_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) -{ - struct bounce_page *bpage; - - if ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) { - /* - * Handle data bouncing. We might also - * want to add support for invalidating - * the caches on broken hardware - */ - dmat->bounce_zone->total_bounced++; - CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x op 0x%x " - "performing bounce", __func__, op, dmat, dmat->flags); - - if (op & BUS_DMASYNC_PREWRITE) { - while (bpage != NULL) { - bcopy((void *)bpage->datavaddr, - (void *)bpage->vaddr, - bpage->datacount); - bpage = STAILQ_NEXT(bpage, links); - } - } - - if (op & BUS_DMASYNC_POSTREAD) { - while (bpage != NULL) { - bcopy((void *)bpage->vaddr, - (void *)bpage->datavaddr, - bpage->datacount); - bpage = STAILQ_NEXT(bpage, links); - } - } - } -} - -void -init_bounce_pages() -{ - - total_bpages = 0; - STAILQ_INIT(&bounce_zone_list); - STAILQ_INIT(&bounce_map_waitinglist); - STAILQ_INIT(&bounce_map_callbacklist); - mtx_init(&bounce_lock, "bounce pages lock", NULL, MTX_DEF); -} - -/* Haiku extension */ -void -uninit_bounce_pages() -{ - /* XXX deep free */ - - mtx_destroy(&bounce_lock); -} - -static struct sysctl_ctx_list * -busdma_sysctl_tree(struct bounce_zone *bz) -{ - return (&bz->sysctl_tree); -} - -static struct sysctl_oid * -busdma_sysctl_tree_top(struct bounce_zone *bz) -{ - return (bz->sysctl_tree_top); -} - -static int -alloc_bounce_zone(bus_dma_tag_t dmat) -{ - struct bounce_zone *bz; - - /* Check to see if we already have a suitable zone */ - STAILQ_FOREACH(bz, &bounce_zone_list, links) { - if ((dmat->alignment <= bz->alignment) - && (dmat->boundary <= bz->boundary) - && (dmat->lowaddr >= bz->lowaddr)) { - dmat->bounce_zone = bz; - return (0); - } - } - - if ((bz = (struct bounce_zone *)malloc(sizeof(*bz), M_DEVBUF, - M_NOWAIT | M_ZERO)) == NULL) - return (ENOMEM); - - STAILQ_INIT(&bz->bounce_page_list); - bz->free_bpages = 0; - bz->reserved_bpages = 0; - bz->active_bpages = 0; - bz->lowaddr = dmat->lowaddr; - bz->alignment = dmat->alignment; - bz->boundary = dmat->boundary; - snprintf(bz->zoneid, 8, "zone%d", busdma_zonecount); - busdma_zonecount++; - snprintf(bz->lowaddrid, 18, "%llx", (uintmax_t)bz->lowaddr); - STAILQ_INSERT_TAIL(&bounce_zone_list, bz, links); - dmat->bounce_zone = bz; - - sysctl_ctx_init(&bz->sysctl_tree); - bz->sysctl_tree_top = SYSCTL_ADD_NODE(&bz->sysctl_tree, - SYSCTL_STATIC_CHILDREN(_hw_busdma), OID_AUTO, bz->zoneid, - CTLFLAG_RD, 0, ""); - if (bz->sysctl_tree_top == NULL) { - sysctl_ctx_free(&bz->sysctl_tree); - return (0); /* XXX error code? */ - } - - SYSCTL_ADD_INT(busdma_sysctl_tree(bz), - SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, - "total_bpages", CTLFLAG_RD, &bz->total_bpages, 0, - "Total bounce pages"); - SYSCTL_ADD_INT(busdma_sysctl_tree(bz), - SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, - "free_bpages", CTLFLAG_RD, &bz->free_bpages, 0, - "Free bounce pages"); - SYSCTL_ADD_INT(busdma_sysctl_tree(bz), - SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, - "reserved_bpages", CTLFLAG_RD, &bz->reserved_bpages, 0, - "Reserved bounce pages"); - SYSCTL_ADD_INT(busdma_sysctl_tree(bz), - SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, - "active_bpages", CTLFLAG_RD, &bz->active_bpages, 0, - "Active bounce pages"); - SYSCTL_ADD_INT(busdma_sysctl_tree(bz), - SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, - "total_bounced", CTLFLAG_RD, &bz->total_bounced, 0, - "Total bounce requests"); - SYSCTL_ADD_INT(busdma_sysctl_tree(bz), - SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, - "total_deferred", CTLFLAG_RD, &bz->total_deferred, 0, - "Total bounce requests that were deferred"); - SYSCTL_ADD_STRING(busdma_sysctl_tree(bz), - SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, - "lowaddr", CTLFLAG_RD, bz->lowaddrid, 0, ""); - SYSCTL_ADD_INT(busdma_sysctl_tree(bz), - SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, - "alignment", CTLFLAG_RD, &bz->alignment, 0, ""); - SYSCTL_ADD_INT(busdma_sysctl_tree(bz), - SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, - "boundary", CTLFLAG_RD, &bz->boundary, 0, ""); - - return (0); -} - -static int -alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages) -{ - struct bounce_zone *bz; - int count; - - bz = dmat->bounce_zone; - count = 0; - while (numpages > 0) { - struct bounce_page *bpage; - - bpage = (struct bounce_page *)malloc(sizeof(*bpage), M_DEVBUF, - M_NOWAIT | M_ZERO); - - if (bpage == NULL) - break; - bpage->vaddr = (vm_offset_t)contigmalloc(PAGE_SIZE, M_DEVBUF, - M_NOWAIT, 0ul, - bz->lowaddr, - PAGE_SIZE, - bz->boundary); - if (bpage->vaddr == 0) { - free(bpage, M_DEVBUF); - break; - } - bpage->busaddr = pmap_kextract(bpage->vaddr); - mtx_lock(&bounce_lock); - STAILQ_INSERT_TAIL(&bz->bounce_page_list, bpage, links); - total_bpages++; - bz->total_bpages++; - bz->free_bpages++; - mtx_unlock(&bounce_lock); - count++; - numpages--; - } - return (count); -} - -static int -reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int commit) -{ - struct bounce_zone *bz; - int pages; - - mtx_assert(&bounce_lock, MA_OWNED); - bz = dmat->bounce_zone; - pages = MIN(bz->free_bpages, map->pagesneeded - map->pagesreserved); - if (commit == 0 && map->pagesneeded > (map->pagesreserved + pages)) - return (map->pagesneeded - (map->pagesreserved + pages)); - bz->free_bpages -= pages; - bz->reserved_bpages += pages; - map->pagesreserved += pages; - pages = map->pagesneeded - map->pagesreserved; - - return (pages); -} - -static bus_addr_t -add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, vm_offset_t vaddr, - bus_size_t size) -{ - struct bounce_zone *bz; - struct bounce_page *bpage; - - KASSERT(dmat->bounce_zone != NULL, ("no bounce zone in dma tag")); - KASSERT(map != NULL && map != &nobounce_dmamap, - ("add_bounce_page: bad map %p", map)); - - bz = dmat->bounce_zone; - if (map->pagesneeded == 0) - panic("add_bounce_page: map doesn't need any pages"); - map->pagesneeded--; - - if (map->pagesreserved == 0) - panic("add_bounce_page: map doesn't need any pages"); - map->pagesreserved--; - - mtx_lock(&bounce_lock); - bpage = STAILQ_FIRST(&bz->bounce_page_list); - if (bpage == NULL) - panic("add_bounce_page: free page list is empty"); - - STAILQ_REMOVE_HEAD(&bz->bounce_page_list, links); - bz->reserved_bpages--; - bz->active_bpages++; - mtx_unlock(&bounce_lock); - - bpage->datavaddr = vaddr; - bpage->datacount = size; - STAILQ_INSERT_TAIL(&(map->bpages), bpage, links); - return (bpage->busaddr); -} - -static void -free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage) -{ - struct bus_dmamap *map; - struct bounce_zone *bz; - - bz = dmat->bounce_zone; - bpage->datavaddr = 0; - bpage->datacount = 0; - - mtx_lock(&bounce_lock); - STAILQ_INSERT_HEAD(&bz->bounce_page_list, bpage, links); - bz->free_bpages++; - bz->active_bpages--; - if ((map = STAILQ_FIRST(&bounce_map_waitinglist)) != NULL) { - if (reserve_bounce_pages(map->dmat, map, 1) == 0) { - STAILQ_REMOVE_HEAD(&bounce_map_waitinglist, links); - STAILQ_INSERT_TAIL(&bounce_map_callbacklist, - map, links); - busdma_swi_pending = 1; - bz->total_deferred++; -#if 0 - swi_sched(vm_ih, 0); -#endif - busdma_swi(); - } - } - mtx_unlock(&bounce_lock); -} - -void -busdma_swi(void) -{ - bus_dma_tag_t dmat; - struct bus_dmamap *map; - - mtx_lock(&bounce_lock); - while ((map = STAILQ_FIRST(&bounce_map_callbacklist)) != NULL) { - STAILQ_REMOVE_HEAD(&bounce_map_callbacklist, links); - mtx_unlock(&bounce_lock); - dmat = map->dmat; - (dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_LOCK); - bus_dmamap_load(map->dmat, map, map->buf, map->buflen, - map->callback, map->callback_arg, /*flags*/0); - (dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_UNLOCK); - mtx_lock(&bounce_lock); - } - mtx_unlock(&bounce_lock); -}