hnddma.c

Revision 1, 35.9 kB (checked in by brainslayer, 4 years ago)
initial checkin
Property svn:executable set to

Line
1	/*
2	* Generic Broadcom Home Networking Division (HND) DMA module.
3	* This supports the following chips: BCM42xx, 44xx, 47xx .
4	*
5	* Copyright 2005, Broadcom Corporation
6	* All Rights Reserved.
7	*
8	* THIS SOFTWARE IS OFFERED "AS IS", AND BROADCOM GRANTS NO WARRANTIES OF ANY
9	* KIND, EXPRESS OR IMPLIED, BY STATUTE, COMMUNICATION OR OTHERWISE. BROADCOM
10	* SPECIFICALLY DISCLAIMS ANY IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS
11	* FOR A SPECIFIC PURPOSE OR NONINFRINGEMENT CONCERNING THIS SOFTWARE.
12	*
13	* $Id$
14	*/
15
16	#include <typedefs.h>
17	#include <osl.h>
18	#include <bcmendian.h>
19	#include <sbconfig.h>
20	#include <bcmutils.h>
21	#include <bcmdevs.h>
22	#include <sbutils.h>
23
24	struct dma_info; /* forward declaration */
25	#define di_t struct dma_info
26
27	#include <sbhnddma.h>
28	#include <hnddma.h>
29
30	/* debug/trace */
31	#define DMA_ERROR(args)
32	#define DMA_TRACE(args)
33
34	/* default dma message level (if input msg_level pointer is null in dma_attach()) */
35	static uint dma_msg_level =
36	0;
37
38	#define MAXNAMEL 8
39
40	/* dma engine software state */
41	typedef struct dma_info {
42	hnddma_t hnddma; /* exported structure */
43	uint msg_level; / message level pointer */
44	char name[MAXNAMEL]; /* callers name for diag msgs */
45
46	void osh; / os handle */
47	sb_t sbh; / sb handle */
48
49	bool dma64; /* dma64 enabled */
50	bool addrext; /* this dma engine supports DmaExtendedAddrChanges */
51
52	dma32regs_t d32txregs; / 32 bits dma tx engine registers */
53	dma32regs_t d32rxregs; / 32 bits dma rx engine registers */
54	dma64regs_t d64txregs; / 64 bits dma tx engine registers */
55	dma64regs_t d64rxregs; / 64 bits dma rx engine registers */
56
57	uint32 dma64align; /* either 8k or 4k depends on number of dd */
58	dma32dd_t txd32; / pointer to dma32 tx descriptor ring */
59	dma64dd_t txd64; / pointer to dma64 tx descriptor ring */
60	uint ntxd; /* # tx descriptors tunable */
61	uint txin; /* index of next descriptor to reclaim */
62	uint txout; /* index of next descriptor to post */
63	uint txavail; /* # free tx descriptors */
64	void *txp; / pointer to parallel array of pointers to packets */
65	ulong txdpa; /* physical address of descriptor ring */
66	uint txdalign; /* #bytes added to alloc'd mem to align txd */
67	uint txdalloc; /* #bytes allocated for the ring */
68
69	dma32dd_t rxd32; / pointer to dma32 rx descriptor ring */
70	dma64dd_t rxd64; / pointer to dma64 rx descriptor ring */
71	uint nrxd; /* # rx descriptors tunable */
72	uint rxin; /* index of next descriptor to reclaim */
73	uint rxout; /* index of next descriptor to post */
74	void *rxp; / pointer to parallel array of pointers to packets */
75	ulong rxdpa; /* physical address of descriptor ring */
76	uint rxdalign; /* #bytes added to alloc'd mem to align rxd */
77	uint rxdalloc; /* #bytes allocated for the ring */
78
79	/* tunables */
80	uint rxbufsize; /* rx buffer size in bytes */
81	uint nrxpost; /* # rx buffers to keep posted */
82	uint rxoffset; /* rxcontrol offset */
83	uint ddoffsetlow; /* add to get dma address of descriptor ring, low 32 bits */
84	uint ddoffsethigh; /* add to get dma address of descriptor ring, high 32 bits */
85	uint dataoffsetlow; /* add to get dma address of data buffer, low 32 bits */
86	uint dataoffsethigh; /* add to get dma address of data buffer, high 32 bits */
87	} dma_info_t;
88
89	#ifdef BCMDMA64
90	#define DMA64_ENAB(di) ((di)->dma64)
91	#else
92	#define DMA64_ENAB(di) (0)
93	#endif
94
95	/* descriptor bumping macros */
96	#define XXD(x, n) ((x) & ((n) - 1))
97	#define TXD(x) XXD((x), di->ntxd)
98	#define RXD(x) XXD((x), di->nrxd)
99	#define NEXTTXD(i) TXD(i + 1)
100	#define PREVTXD(i) TXD(i - 1)
101	#define NEXTRXD(i) RXD(i + 1)
102	#define NTXDACTIVE(h, t) TXD(t - h)
103	#define NRXDACTIVE(h, t) RXD(t - h)
104
105	/* macros to convert between byte offsets and indexes */
106	#define B2I(bytes, type) ((bytes) / sizeof(type))
107	#define I2B(index, type) ((index) * sizeof(type))
108
109	#define PCI32ADDR_HIGH 0xc0000000 /* address[31:30] */
110	#define PCI32ADDR_HIGH_SHIFT 30
111
112
113	/* prototypes */
114	static bool dma_isaddrext(dma_info_t *di);
115	static bool dma_alloc(dma_info_t *di, uint direction);
116
117	static bool dma32_alloc(dma_info_t *di, uint direction);
118	static void dma32_txreset(dma_info_t *di);
119	static void dma32_rxreset(dma_info_t *di);
120	static bool dma32_txsuspendedidle(dma_info_t *di);
121	static int dma32_txfast(dma_info_t di, void p0, uint32 coreflags);
122	static void* dma32_getnexttxp(dma_info_t *di, bool forceall);
123	static void* dma32_getnextrxp(dma_info_t *di, bool forceall);
124	static void dma32_txrotate(di_t *di);
125
126	/* prototype or stubs */
127	#ifdef BCMDMA64
128	static bool dma64_alloc(dma_info_t *di, uint direction);
129	static void dma64_txreset(dma_info_t *di);
130	static void dma64_rxreset(dma_info_t *di);
131	static bool dma64_txsuspendedidle(dma_info_t *di);
132	static int dma64_txfast(dma_info_t di, void p0, uint32 coreflags);
133	static void* dma64_getnexttxp(dma_info_t *di, bool forceall);
134	static void* dma64_getnextrxp(dma_info_t *di, bool forceall);
135	static void dma64_txrotate(di_t *di);
136	#else
137	static bool dma64_alloc(dma_info_t *di, uint direction) { return TRUE; }
138	static void dma64_txreset(dma_info_t *di) {}
139	static void dma64_rxreset(dma_info_t *di) {}
140	static bool dma64_txsuspendedidle(dma_info_t *di) { return TRUE;}
141	static int dma64_txfast(dma_info_t di, void p0, uint32 coreflags) { return 0; }
142	static void* dma64_getnexttxp(dma_info_t *di, bool forceall) { return NULL; }
143	static void* dma64_getnextrxp(dma_info_t *di, bool forceall) { return NULL; }
144	static void dma64_txrotate(di_t *di) { return; }
145	#endif
146
147
148
149	void*
150	dma_attach(osl_t osh, char name, sb_t sbh, void dmaregstx, void *dmaregsrx,
151	uint ntxd, uint nrxd, uint rxbufsize, uint nrxpost, uint rxoffset, uint *msg_level)
152	{
153	dma_info_t *di;
154	uint size;
155
156	/* allocate private info structure */
157	if ((di = MALLOC(osh, sizeof (dma_info_t))) == NULL) {
158	return (NULL);
159	}
160	bzero((char*)di, sizeof (dma_info_t));
161
162	di->msg_level = msg_level ? msg_level : &dma_msg_level;
163
164	if (sbh != NULL)
165	di->dma64 = ((sb_coreflagshi(sbh, 0, 0) & SBTMH_DMA64) == SBTMH_DMA64);
166
167	#ifndef BCMDMA64
168	if (di->dma64) {
169	DMA_ERROR(("dma_attach: driver doesn't have the capability to support 64 bits DMA\n"));
170	goto fail;
171	}
172	#endif
173
174	/* check arguments */
175	ASSERT(ISPOWEROF2(ntxd));
176	ASSERT(ISPOWEROF2(nrxd));
177	if (nrxd == 0)
178	ASSERT(dmaregsrx == NULL);
179	if (ntxd == 0)
180	ASSERT(dmaregstx == NULL);
181
182
183	/* init dma reg pointer */
184	if (di->dma64) {
185	ASSERT(ntxd <= D64MAXDD);
186	ASSERT(nrxd <= D64MAXDD);
187	di->d64txregs = (dma64regs_t *)dmaregstx;
188	di->d64rxregs = (dma64regs_t *)dmaregsrx;
189
190	di->dma64align = D64RINGALIGN;
191	if ((ntxd < D64MAXDD / 2) && (nrxd < D64MAXDD / 2)) {
192	/* for smaller dd table, HW relax the alignment requirement */
193	di->dma64align = D64RINGALIGN / 2;
194	}
195	} else {
196	ASSERT(ntxd <= D32MAXDD);
197	ASSERT(nrxd <= D32MAXDD);
198	di->d32txregs = (dma32regs_t *)dmaregstx;
199	di->d32rxregs = (dma32regs_t *)dmaregsrx;
200	}
201
202
203	/* make a private copy of our callers name */
204	strncpy(di->name, name, MAXNAMEL);
205	di->name[MAXNAMEL-1] = '\0';
206
207	di->osh = osh;
208	di->sbh = sbh;
209
210	/* save tunables */
211	di->ntxd = ntxd;
212	di->nrxd = nrxd;
213	di->rxbufsize = rxbufsize;
214	di->nrxpost = nrxpost;
215	di->rxoffset = rxoffset;
216
217	/*
218	* figure out the DMA physical address offset for dd and data
219	* for old chips w/o sb, use zero
220	* for new chips w sb,
221	* PCI/PCIE: they map silicon backplace address to zero based memory, need offset
222	* Other bus: use zero
223	* SB_BUS BIGENDIAN kludge: use sdram swapped region for data buffer, not descriptor
224	*/
225	di->ddoffsetlow = 0;
226	di->dataoffsetlow = 0;
227	if (sbh != NULL) {
228	if (sbh->bustype == PCI_BUS) { /* for pci bus, add offset */
229	if ((sbh->buscoretype == SB_PCIE) && di->dma64){
230	di->ddoffsetlow = 0;
231	di->ddoffsethigh = SB_PCIE_DMA_H32;
232	} else {
233	di->ddoffsetlow = SB_PCI_DMA;
234	di->ddoffsethigh = 0;
235	}
236	di->dataoffsetlow = di->ddoffsetlow;
237	di->dataoffsethigh = di->ddoffsethigh;
238	}
239	#if defined(__mips__) && defined(IL_BIGENDIAN)
240	/* use sdram swapped region for data buffers but not dma descriptors */
241	di->dataoffsetlow = di->dataoffsetlow + SB_SDRAM_SWAPPED;
242	#endif
243	}
244
245	di->addrext = dma_isaddrext(di);
246
247	DMA_TRACE(("%s: dma_attach: osh %p ntxd %d nrxd %d rxbufsize %d nrxpost %d rxoffset %d ddoffset 0x%x dataoffset 0x%x\n",
248	name, osh, ntxd, nrxd, rxbufsize, nrxpost, rxoffset, di->ddoffsetlow, di->dataoffsetlow));
249
250	/* allocate tx packet pointer vector */
251	if (ntxd) {
252	size = ntxd * sizeof (void*);
253	if ((di->txp = MALLOC(osh, size)) == NULL) {
254	DMA_ERROR(("%s: dma_attach: out of tx memory, malloced %d bytes\n", di->name, MALLOCED(osh)));
255	goto fail;
256	}
257	bzero((char*)di->txp, size);
258	}
259
260	/* allocate rx packet pointer vector */
261	if (nrxd) {
262	size = nrxd * sizeof (void*);
263	if ((di->rxp = MALLOC(osh, size)) == NULL) {
264	DMA_ERROR(("%s: dma_attach: out of rx memory, malloced %d bytes\n", di->name, MALLOCED(osh)));
265	goto fail;
266	}
267	bzero((char*)di->rxp, size);
268	}
269
270	/* allocate transmit descriptor ring, only need ntxd descriptors but it must be aligned */
271	if (ntxd) {
272	if (!dma_alloc(di, DMA_TX))
273	goto fail;
274	}
275
276	/* allocate receive descriptor ring, only need nrxd descriptors but it must be aligned */
277	if (nrxd) {
278	if (!dma_alloc(di, DMA_RX))
279	goto fail;
280	}
281
282	if ((di->ddoffsetlow == SB_PCI_DMA) && (di->txdpa > SB_PCI_DMA_SZ) && !di->addrext) {
283	DMA_ERROR(("%s: dma_attach: txdpa 0x%lx: addrext not supported\n", di->name, di->txdpa));
284	goto fail;
285	}
286	if ((di->ddoffsetlow == SB_PCI_DMA) && (di->rxdpa > SB_PCI_DMA_SZ) && !di->addrext) {
287	DMA_ERROR(("%s: dma_attach: rxdpa 0x%lx: addrext not supported\n", di->name, di->rxdpa));
288	goto fail;
289	}
290
291	return ((void*)di);
292
293	fail:
294	dma_detach((void*)di);
295	return (NULL);
296	}
297
298	static bool
299	dma_alloc(dma_info_t *di, uint direction)
300	{
301	if (DMA64_ENAB(di)) {
302	return dma64_alloc(di, direction);
303	} else {
304	return dma32_alloc(di, direction);
305	}
306	}
307
308	/* may be called with core in reset */
309	void
310	dma_detach(dma_info_t *di)
311	{
312	if (di == NULL)
313	return;
314
315	DMA_TRACE(("%s: dma_detach\n", di->name));
316
317	/* shouldn't be here if descriptors are unreclaimed */
318	ASSERT(di->txin == di->txout);
319	ASSERT(di->rxin == di->rxout);
320
321	/* free dma descriptor rings */
322	if (di->txd32)
323	DMA_FREE_CONSISTENT(di->osh, ((int8*)di->txd32 - di->txdalign), di->txdalloc, (di->txdpa - di->txdalign));
324	if (di->rxd32)
325	DMA_FREE_CONSISTENT(di->osh, ((int8*)di->rxd32 - di->rxdalign), di->rxdalloc, (di->rxdpa - di->rxdalign));
326
327	/* free packet pointer vectors */
328	if (di->txp)
329	MFREE(di->osh, (void)di->txp, (di->ntxd sizeof (void*)));
330	if (di->rxp)
331	MFREE(di->osh, (void)di->rxp, (di->nrxd sizeof (void*)));
332
333	/* free our private info structure */
334	MFREE(di->osh, (void*)di, sizeof (dma_info_t));
335	}
336
337	/* return TRUE if this dma engine supports DmaExtendedAddrChanges, otherwise FALSE */
338	static bool
339	dma_isaddrext(dma_info_t *di)
340	{
341	uint32 w;
342
343	if (DMA64_ENAB(di)) {
344	OR_REG(&di->d64txregs->control, D64_XC_AE);
345	w = R_REG(&di->d32txregs->control);
346	AND_REG(&di->d32txregs->control, ~D64_XC_AE);
347	return ((w & XC_AE) == D64_XC_AE);
348	} else {
349	OR_REG(&di->d32txregs->control, XC_AE);
350	w = R_REG(&di->d32txregs->control);
351	AND_REG(&di->d32txregs->control, ~XC_AE);
352	return ((w & XC_AE) == XC_AE);
353	}
354	}
355
356	void
357	dma_txreset(dma_info_t *di)
358	{
359	DMA_TRACE(("%s: dma_txreset\n", di->name));
360
361	if (DMA64_ENAB(di))
362	dma64_txreset(di);
363	else
364	dma32_txreset(di);
365	}
366
367	void
368	dma_rxreset(dma_info_t *di)
369	{
370	DMA_TRACE(("%s: dma_rxreset\n", di->name));
371
372	if (DMA64_ENAB(di))
373	dma64_rxreset(di);
374	else
375	dma32_rxreset(di);
376	}
377
378	/* initialize descriptor table base address */
379	static void
380	dma_ddtable_init(dma_info_t *di, uint direction, ulong pa)
381	{
382	if (DMA64_ENAB(di)) {
383	if (direction == DMA_TX) {
384	W_REG(&di->d64txregs->addrlow, pa + di->ddoffsetlow);
385	W_REG(&di->d64txregs->addrhigh, di->ddoffsethigh);
386	} else {
387	W_REG(&di->d64rxregs->addrlow, pa + di->ddoffsetlow);
388	W_REG(&di->d64rxregs->addrhigh, di->ddoffsethigh);
389	}
390	} else {
391	uint32 offset = di->ddoffsetlow;
392	if ((offset != SB_PCI_DMA) \|\| !(pa & PCI32ADDR_HIGH)) {
393	if (direction == DMA_TX)
394	W_REG(&di->d32txregs->addr, (pa + offset));
395	else
396	W_REG(&di->d32rxregs->addr, (pa + offset));
397	} else {
398	/* dma32 address extension */
399	uint32 ae;
400	ASSERT(di->addrext);
401	ae = (pa & PCI32ADDR_HIGH) >> PCI32ADDR_HIGH_SHIFT;
402
403	if (direction == DMA_TX) {
404	W_REG(&di->d32txregs->addr, ((pa & ~PCI32ADDR_HIGH) + offset));
405	SET_REG(&di->d32txregs->control, XC_AE, (ae << XC_AE_SHIFT));
406	} else {
407	W_REG(&di->d32rxregs->addr, ((pa & ~PCI32ADDR_HIGH) + offset));
408	SET_REG(&di->d32rxregs->control, RC_AE, (ae << RC_AE_SHIFT));
409	}
410	}
411	}
412	}
413
414	/* init the tx or rx descriptor */
415	static INLINE void
416	dma32_dd_upd(dma_info_t di, dma32dd_t ddring, ulong pa, uint outidx, uint32 *ctrl)
417	{
418	uint offset = di->dataoffsetlow;
419
420	if ((offset != SB_PCI_DMA) \|\| !(pa & PCI32ADDR_HIGH)) {
421	W_SM(&ddring[outidx].addr, BUS_SWAP32(pa + offset));
422	W_SM(&ddring[outidx].ctrl, BUS_SWAP32(*ctrl));
423	} else {
424	/* address extension */
425	uint32 ae;
426	ASSERT(di->addrext);
427	ae = (pa & PCI32ADDR_HIGH) >> PCI32ADDR_HIGH_SHIFT;
428
429	*ctrl \|= (ae << CTRL_AE_SHIFT);
430	W_SM(&ddring[outidx].addr, BUS_SWAP32((pa & ~PCI32ADDR_HIGH) + offset));
431	W_SM(&ddring[outidx].ctrl, BUS_SWAP32(*ctrl));
432	}
433	}
434
435	/* init the tx or rx descriptor */
436	static INLINE void
437	dma64_dd_upd(dma_info_t di, dma64dd_t ddring, ulong pa, uint outidx, uint32 *flags, uint32 bufcount)
438	{
439	uint32 bufaddr_low = pa + di->dataoffsetlow;
440	uint32 bufaddr_high = 0 + di->dataoffsethigh;
441
442	uint32 ctrl2 = bufcount & D64_CTRL2_BC_MASK;
443
444	W_SM(&ddring[outidx].addrlow, BUS_SWAP32(bufaddr_low));
445	W_SM(&ddring[outidx].addrhigh, BUS_SWAP32(bufaddr_high));
446	W_SM(&ddring[outidx].ctrl1, BUS_SWAP32(*flags));
447	W_SM(&ddring[outidx].ctrl2, BUS_SWAP32(ctrl2));
448	}
449
450	void
451	dma_txinit(dma_info_t *di)
452	{
453	DMA_TRACE(("%s: dma_txinit\n", di->name));
454
455	di->txin = di->txout = 0;
456	di->txavail = di->ntxd - 1;
457
458	/* clear tx descriptor ring */
459	if (DMA64_ENAB(di)) {
460	BZERO_SM((void)di->txd64, (di->ntxd sizeof (dma64dd_t)));
461	W_REG(&di->d64txregs->control, XC_XE);
462	dma_ddtable_init(di, DMA_TX, di->txdpa);
463	} else {
464	BZERO_SM((void)di->txd32, (di->ntxd sizeof (dma32dd_t)));
465	W_REG(&di->d32txregs->control, XC_XE);
466	dma_ddtable_init(di, DMA_TX, di->txdpa);
467	}
468	}
469
470	bool
471	dma_txenabled(dma_info_t *di)
472	{
473	uint32 xc;
474
475	/* If the chip is dead, it is not enabled :-) */
476	if (DMA64_ENAB(di)) {
477	xc = R_REG(&di->d64txregs->control);
478	return ((xc != 0xffffffff) && (xc & D64_XC_XE));
479	} else {
480	xc = R_REG(&di->d32txregs->control);
481	return ((xc != 0xffffffff) && (xc & XC_XE));
482	}
483	}
484
485	void
486	dma_txsuspend(dma_info_t *di)
487	{
488	DMA_TRACE(("%s: dma_txsuspend\n", di->name));
489	if (DMA64_ENAB(di))
490	OR_REG(&di->d64txregs->control, D64_XC_SE);
491	else
492	OR_REG(&di->d32txregs->control, XC_SE);
493	}
494
495	void
496	dma_txresume(dma_info_t *di)
497	{
498	DMA_TRACE(("%s: dma_txresume\n", di->name));
499	if (DMA64_ENAB(di))
500	AND_REG(&di->d64txregs->control, ~D64_XC_SE);
501	else
502	AND_REG(&di->d32txregs->control, ~XC_SE);
503	}
504
505	bool
506	dma_txsuspendedidle(dma_info_t *di)
507	{
508	if (DMA64_ENAB(di))
509	return dma64_txsuspendedidle(di);
510	else
511	return dma32_txsuspendedidle(di);
512	}
513
514	bool
515	dma_txsuspended(dma_info_t *di)
516	{
517	if (DMA64_ENAB(di))
518	return ((R_REG(&di->d64txregs->control) & D64_XC_SE) == D64_XC_SE);
519	else
520	return ((R_REG(&di->d32txregs->control) & XC_SE) == XC_SE);
521	}
522
523	bool
524	dma_txstopped(dma_info_t *di)
525	{
526	if (DMA64_ENAB(di))
527	return ((R_REG(&di->d64txregs->status0) & D64_XS0_XS_MASK) == D64_XS0_XS_STOPPED);
528	else
529	return ((R_REG(&di->d32txregs->status) & XS_XS_MASK) == XS_XS_STOPPED);
530	}
531
532	bool
533	dma_rxstopped(dma_info_t *di)
534	{
535	if (DMA64_ENAB(di))
536	return ((R_REG(&di->d64rxregs->status0) & D64_RS0_RS_MASK) == D64_RS0_RS_STOPPED);
537	else
538	return ((R_REG(&di->d32rxregs->status) & RS_RS_MASK) == RS_RS_STOPPED);
539	}
540
541	void
542	dma_fifoloopbackenable(dma_info_t *di)
543	{
544	DMA_TRACE(("%s: dma_fifoloopbackenable\n", di->name));
545	if (DMA64_ENAB(di))
546	OR_REG(&di->d64txregs->control, D64_XC_LE);
547	else
548	OR_REG(&di->d32txregs->control, XC_LE);
549	}
550
551	void
552	dma_rxinit(dma_info_t *di)
553	{
554	DMA_TRACE(("%s: dma_rxinit\n", di->name));
555
556	di->rxin = di->rxout = 0;
557
558	/* clear rx descriptor ring */
559	if (DMA64_ENAB(di)) {
560	BZERO_SM((void)di->rxd64, (di->nrxd sizeof (dma64dd_t)));
561	dma_rxenable(di);
562	dma_ddtable_init(di, DMA_RX, di->rxdpa);
563	} else {
564	BZERO_SM((void)di->rxd32, (di->nrxd sizeof (dma32dd_t)));
565	dma_rxenable(di);
566	dma_ddtable_init(di, DMA_RX, di->rxdpa);
567	}
568	}
569
570	void
571	dma_rxenable(dma_info_t *di)
572	{
573	DMA_TRACE(("%s: dma_rxenable\n", di->name));
574	if (DMA64_ENAB(di))
575	W_REG(&di->d64rxregs->control, ((di->rxoffset << D64_RC_RO_SHIFT) \| D64_RC_RE));
576	else
577	W_REG(&di->d32rxregs->control, ((di->rxoffset << RC_RO_SHIFT) \| RC_RE));
578	}
579
580	bool
581	dma_rxenabled(dma_info_t *di)
582	{
583	uint32 rc;
584
585	if (DMA64_ENAB(di)) {
586	rc = R_REG(&di->d64rxregs->control);
587	return ((rc != 0xffffffff) && (rc & D64_RC_RE));
588	} else {
589	rc = R_REG(&di->d32rxregs->control);
590	return ((rc != 0xffffffff) && (rc & RC_RE));
591	}
592	}
593
594
595	/* !! tx entry routine */
596	int
597	dma_txfast(dma_info_t di, void p0, uint32 coreflags)
598	{
599	if (DMA64_ENAB(di)) {
600	return dma64_txfast(di, p0, coreflags);
601	} else {
602	return dma32_txfast(di, p0, coreflags);
603	}
604	}
605
606	/* !! rx entry routine, returns a pointer to the next frame received, or NULL if there are no more */
607	void*
608	dma_rx(dma_info_t *di)
609	{
610	void *p;
611	uint len;
612	int skiplen = 0;
613
614	while ((p = dma_getnextrxp(di, FALSE))) {
615	/* skip giant packets which span multiple rx descriptors */
616	if (skiplen > 0) {
617	skiplen -= di->rxbufsize;
618	if (skiplen < 0)
619	skiplen = 0;
620	PKTFREE(di->osh, p, FALSE);
621	continue;
622	}
623
624	len = ltoh16((uint16)(PKTDATA(di->osh, p)));
625	DMA_TRACE(("%s: dma_rx len %d\n", di->name, len));
626
627	/* bad frame length check */
628	if (len > (di->rxbufsize - di->rxoffset)) {
629	DMA_ERROR(("%s: dma_rx: bad frame length (%d)\n", di->name, len));
630	if (len > 0)
631	skiplen = len - (di->rxbufsize - di->rxoffset);
632	PKTFREE(di->osh, p, FALSE);
633	di->hnddma.rxgiants++;
634	continue;
635	}
636
637	/* set actual length */
638	PKTSETLEN(di->osh, p, (di->rxoffset + len));
639
640	break;
641	}
642
643	return (p);
644	}
645
646	/* post receive buffers */
647	void
648	dma_rxfill(dma_info_t *di)
649	{
650	void *p;
651	uint rxin, rxout;
652	uint32 ctrl;
653	uint n;
654	uint i;
655	uint32 pa;
656	uint rxbufsize;
657
658	/*
659	* Determine how many receive buffers we're lacking
660	* from the full complement, allocate, initialize,
661	* and post them, then update the chip rx lastdscr.
662	*/
663
664	rxin = di->rxin;
665	rxout = di->rxout;
666	rxbufsize = di->rxbufsize;
667
668	n = di->nrxpost - NRXDACTIVE(rxin, rxout);
669
670	DMA_TRACE(("%s: dma_rxfill: post %d\n", di->name, n));
671
672	for (i = 0; i < n; i++) {
673	if ((p = PKTGET(di->osh, rxbufsize, FALSE)) == NULL) {
674	DMA_ERROR(("%s: dma_rxfill: out of rxbufs\n", di->name));
675	di->hnddma.rxnobuf++;
676	break;
677	}
678
679	/* Do a cached write instead of uncached write since DMA_MAP
680	* will flush the cache. */
681	(uint32)(PKTDATA(di->osh, p)) = 0;
682
683	pa = (uint32) DMA_MAP(di->osh, PKTDATA(di->osh, p), rxbufsize, DMA_RX, p);
684	ASSERT(ISALIGNED(pa, 4));
685
686	/* save the free packet pointer */
687	ASSERT(di->rxp[rxout] == NULL);
688	di->rxp[rxout] = p;
689
690	if (DMA64_ENAB(di)) {
691	/* prep the descriptor control value */
692	if (rxout == (di->nrxd - 1))
693	ctrl = CTRL_EOT;
694
695	dma64_dd_upd(di, di->rxd64, pa, rxout, &ctrl, rxbufsize);
696	} else {
697	/* prep the descriptor control value */
698	ctrl = rxbufsize;
699	if (rxout == (di->nrxd - 1))
700	ctrl \|= CTRL_EOT;
701	dma32_dd_upd(di, di->rxd32, pa, rxout, &ctrl);
702	}
703
704	rxout = NEXTRXD(rxout);
705	}
706
707	di->rxout = rxout;
708
709	/* update the chip lastdscr pointer */
710	if (DMA64_ENAB(di)) {
711	W_REG(&di->d64rxregs->ptr, I2B(rxout, dma64dd_t));
712	} else {
713	W_REG(&di->d32rxregs->ptr, I2B(rxout, dma32dd_t));
714	}
715	}
716
717	void
718	dma_txreclaim(dma_info_t *di, bool forceall)
719	{
720	void *p;
721
722	DMA_TRACE(("%s: dma_txreclaim %s\n", di->name, forceall ? "all" : ""));
723
724	while ((p = dma_getnexttxp(di, forceall)))
725	PKTFREE(di->osh, p, TRUE);
726	}
727
728	/*
729	* Reclaim next completed txd (txds if using chained buffers) and
730	* return associated packet.
731	* If 'force' is true, reclaim txd(s) and return associated packet
732	* regardless of the value of the hardware "curr" pointer.
733	*/
734	void*
735	dma_getnexttxp(dma_info_t *di, bool forceall)
736	{
737	if (DMA64_ENAB(di)) {
738	return dma64_getnexttxp(di, forceall);
739	} else {
740	return dma32_getnexttxp(di, forceall);
741	}
742	}
743
744	/* like getnexttxp but no reclaim */
745	void*
746	dma_peeknexttxp(dma_info_t *di)
747	{
748	uint end, i;
749
750	if (DMA64_ENAB(di)) {
751	end = B2I(R_REG(&di->d64txregs->status0) & D64_XS0_CD_MASK, dma64dd_t);
752	} else {
753	end = B2I(R_REG(&di->d32txregs->status) & XS_CD_MASK, dma32dd_t);
754	}
755
756	for (i = di->txin; i != end; i = NEXTTXD(i))
757	if (di->txp[i])
758	return (di->txp[i]);
759
760	return (NULL);
761	}
762
763	/*
764	* Rotate all active tx dma ring entries "forward" by (ActiveDescriptor - txin).
765	*/
766	void
767	dma_txrotate(di_t *di)
768	{
769	if (DMA64_ENAB(di)) {
770	dma64_txrotate(di);
771	} else {
772	dma32_txrotate(di);
773	}
774	}
775
776	void
777	dma_rxreclaim(dma_info_t *di)
778	{
779	void *p;
780
781	DMA_TRACE(("%s: dma_rxreclaim\n", di->name));
782
783	while ((p = dma_getnextrxp(di, TRUE)))
784	PKTFREE(di->osh, p, FALSE);
785	}
786
787	void *
788	dma_getnextrxp(dma_info_t *di, bool forceall)
789	{
790	if (DMA64_ENAB(di)) {
791	return dma64_getnextrxp(di, forceall);
792	} else {
793	return dma32_getnextrxp(di, forceall);
794	}
795	}
796
797	uintptr
798	dma_getvar(dma_info_t di, char name)
799	{
800	if (!strcmp(name, "&txavail"))
801	return ((uintptr) &di->txavail);
802	else {
803	ASSERT(0);
804	}
805	return (0);
806	}
807
808	void
809	dma_txblock(dma_info_t *di)
810	{
811	di->txavail = 0;
812	}
813
814	void
815	dma_txunblock(dma_info_t *di)
816	{
817	di->txavail = di->ntxd - NTXDACTIVE(di->txin, di->txout) - 1;
818	}
819
820	uint
821	dma_txactive(dma_info_t *di)
822	{
823	return (NTXDACTIVE(di->txin, di->txout));
824	}
825
826	void
827	dma_rxpiomode(dma32regs_t *regs)
828	{
829	W_REG(&regs->control, RC_FM);
830	}
831
832	void
833	dma_txpioloopback(dma32regs_t *regs)
834	{
835	OR_REG(&regs->control, XC_LE);
836	}
837
838
839
840
841	/* 32 bits DMA non-inline functions */
842	static bool
843	dma32_alloc(dma_info_t *di, uint direction)
844	{
845	uint size;
846	uint ddlen;
847	void *va;
848
849	ddlen = sizeof (dma32dd_t);
850
851	size = (direction == DMA_TX) ? (di->ntxd * ddlen) : (di->nrxd * ddlen);
852
853	if (!ISALIGNED(DMA_CONSISTENT_ALIGN, D32RINGALIGN))
854	size += D32RINGALIGN;
855
856
857	if (direction == DMA_TX) {
858	if ((va = DMA_ALLOC_CONSISTENT(di->osh, size, &di->txdpa)) == NULL) {
859	DMA_ERROR(("%s: dma_attach: DMA_ALLOC_CONSISTENT(ntxd) failed\n", di->name));
860	return FALSE;
861	}
862
863	di->txd32 = (dma32dd_t*) ROUNDUP((uintptr)va, D32RINGALIGN);
864	di->txdalign = (uint)((int8)di->txd32 - (int8)va);
865	di->txdpa += di->txdalign;
866	di->txdalloc = size;
867	ASSERT(ISALIGNED((uintptr)di->txd32, D32RINGALIGN));
868	} else {
869	if ((va = DMA_ALLOC_CONSISTENT(di->osh, size, &di->rxdpa)) == NULL) {
870	DMA_ERROR(("%s: dma_attach: DMA_ALLOC_CONSISTENT(nrxd) failed\n", di->name));
871	return FALSE;
872	}
873	di->rxd32 = (dma32dd_t*) ROUNDUP((uintptr)va, D32RINGALIGN);
874	di->rxdalign = (uint)((int8)di->rxd32 - (int8)va);
875	di->rxdpa += di->rxdalign;
876	di->rxdalloc = size;
877	ASSERT(ISALIGNED((uintptr)di->rxd32, D32RINGALIGN));
878	}
879
880	return TRUE;
881	}
882
883	static void
884	dma32_txreset(dma_info_t *di)
885	{
886	uint32 status;
887
888	/* suspend tx DMA first */
889	W_REG(&di->d32txregs->control, XC_SE);
890	SPINWAIT((status = (R_REG(&di->d32txregs->status) & XS_XS_MASK)) != XS_XS_DISABLED &&
891	status != XS_XS_IDLE &&
892	status != XS_XS_STOPPED,
893	10000);
894
895	W_REG(&di->d32txregs->control, 0);
896	SPINWAIT((status = (R_REG(&di->d32txregs->status) & XS_XS_MASK)) != XS_XS_DISABLED,
897	10000);
898
899	if (status != XS_XS_DISABLED) {
900	DMA_ERROR(("%s: dma_txreset: dma cannot be stopped\n", di->name));
901	}
902
903	/* wait for the last transaction to complete */
904	OSL_DELAY(300);
905	}
906
907	static void
908	dma32_rxreset(dma_info_t *di)
909	{
910	uint32 status;
911
912	W_REG(&di->d32rxregs->control, 0);
913	SPINWAIT((status = (R_REG(&di->d32rxregs->status) & RS_RS_MASK)) != RS_RS_DISABLED,
914	10000);
915
916	if (status != RS_RS_DISABLED) {
917	DMA_ERROR(("%s: dma_rxreset: dma cannot be stopped\n", di->name));
918	}
919	}
920
921	static bool
922	dma32_txsuspendedidle(dma_info_t *di)
923	{
924	if (!(R_REG(&di->d32txregs->control) & XC_SE))
925	return 0;
926
927	if ((R_REG(&di->d32txregs->status) & XS_XS_MASK) != XS_XS_IDLE)
928	return 0;
929
930	OSL_DELAY(2);
931	return ((R_REG(&di->d32txregs->status) & XS_XS_MASK) == XS_XS_IDLE);
932	}
933
934	/*
935	* supports full 32bit dma engine buffer addressing so
936	* dma buffers can cross 4 Kbyte page boundaries.
937	*/
938	static int
939	dma32_txfast(dma_info_t di, void p0, uint32 coreflags)
940	{
941	void p, next;
942	uchar *data;
943	uint len;
944	uint txout;
945	uint32 ctrl;
946	uint32 pa;
947
948	DMA_TRACE(("%s: dma_txfast\n", di->name));
949
950	txout = di->txout;
951	ctrl = 0;
952
953	/*
954	* Walk the chain of packet buffers
955	* allocating and initializing transmit descriptor entries.
956	*/
957	for (p = p0; p; p = next) {
958	data = PKTDATA(di->osh, p);
959	len = PKTLEN(di->osh, p);
960	next = PKTNEXT(di->osh, p);
961
962	/* return nonzero if out of tx descriptors */
963	if (NEXTTXD(txout) == di->txin)
964	goto outoftxd;
965
966	if (len == 0)
967	continue;
968
969	/* get physical address of buffer start */
970	pa = (uint32) DMA_MAP(di->osh, data, len, DMA_TX, p);
971
972	/* build the descriptor control value */
973	ctrl = len & CTRL_BC_MASK;
974
975	ctrl \|= coreflags;
976
977	if (p == p0)
978	ctrl \|= CTRL_SOF;
979	if (next == NULL)
980	ctrl \|= (CTRL_IOC \| CTRL_EOF);
981	if (txout == (di->ntxd - 1))
982	ctrl \|= CTRL_EOT;
983
984	if (DMA64_ENAB(di)) {
985	dma64_dd_upd(di, di->txd64, pa, txout, &ctrl, len);
986	} else {
987	dma32_dd_upd(di, di->txd32, pa, txout, &ctrl);
988	}
989
990	ASSERT(di->txp[txout] == NULL);
991
992	txout = NEXTTXD(txout);
993	}
994
995	/* if last txd eof not set, fix it */
996	if (!(ctrl & CTRL_EOF))
997	W_SM(&di->txd32[PREVTXD(txout)].ctrl, BUS_SWAP32(ctrl \| CTRL_IOC \| CTRL_EOF));
998
999	/* save the packet */
1000	di->txp[PREVTXD(txout)] = p0;
1001
1002	/* bump the tx descriptor index */
1003	di->txout = txout;
1004
1005	/* kick the chip */
1006	if (DMA64_ENAB(di)) {
1007	W_REG(&di->d64txregs->ptr, I2B(txout, dma64dd_t));
1008	} else {
1009	W_REG(&di->d32txregs->ptr, I2B(txout, dma32dd_t));
1010	}
1011
1012	/* tx flow control */
1013	di->txavail = di->ntxd - NTXDACTIVE(di->txin, di->txout) - 1;
1014
1015	return (0);
1016
1017	outoftxd:
1018	DMA_ERROR(("%s: dma_txfast: out of txds\n", di->name));
1019	PKTFREE(di->osh, p0, TRUE);
1020	di->txavail = 0;
1021	di->hnddma.txnobuf++;
1022	return (-1);
1023	}
1024
1025	static void*
1026	dma32_getnexttxp(dma_info_t *di, bool forceall)
1027	{
1028	uint start, end, i;
1029	void *txp;
1030
1031	DMA_TRACE(("%s: dma_getnexttxp %s\n", di->name, forceall ? "all" : ""));
1032
1033	txp = NULL;
1034
1035	start = di->txin;
1036	if (forceall)
1037	end = di->txout;
1038	else
1039	end = B2I(R_REG(&di->d32txregs->status) & XS_CD_MASK, dma32dd_t);
1040
1041	if ((start == 0) && (end > di->txout))
1042	goto bogus;
1043
1044	for (i = start; i != end && !txp; i = NEXTTXD(i)) {
1045	DMA_UNMAP(di->osh, (BUS_SWAP32(R_SM(&di->txd32[i].addr)) - di->dataoffsetlow),
1046	(BUS_SWAP32(R_SM(&di->txd32[i].ctrl)) & CTRL_BC_MASK), DMA_TX, di->txp[i]);
1047
1048	W_SM(&di->txd32[i].addr, 0xdeadbeef);
1049	txp = di->txp[i];
1050	di->txp[i] = NULL;
1051	}
1052
1053	di->txin = i;
1054
1055	/* tx flow control */
1056	di->txavail = di->ntxd - NTXDACTIVE(di->txin, di->txout) - 1;
1057
1058	return (txp);
1059
1060	bogus:
1061	/*
1062	DMA_ERROR(("dma_getnexttxp: bogus curr: start %d end %d txout %d force %d\n",
1063	start, end, di->txout, forceall));
1064	*/
1065	return (NULL);
1066	}
1067
1068	static void *
1069	dma32_getnextrxp(dma_info_t *di, bool forceall)
1070	{
1071	uint i;
1072	void *rxp;
1073
1074	/* if forcing, dma engine must be disabled */
1075	ASSERT(!forceall \|\| !dma_rxenabled(di));
1076
1077	i = di->rxin;
1078
1079	/* return if no packets posted */
1080	if (i == di->rxout)
1081	return (NULL);
1082
1083	/* ignore curr if forceall */
1084	if (!forceall && (i == B2I(R_REG(&di->d32rxregs->status) & RS_CD_MASK, dma32dd_t)))
1085	return (NULL);
1086
1087	/* get the packet pointer that corresponds to the rx descriptor */
1088	rxp = di->rxp[i];
1089	ASSERT(rxp);
1090	di->rxp[i] = NULL;
1091
1092	/* clear this packet from the descriptor ring */
1093	DMA_UNMAP(di->osh, (BUS_SWAP32(R_SM(&di->rxd32[i].addr)) - di->dataoffsetlow),
1094	di->rxbufsize, DMA_RX, rxp);
1095	W_SM(&di->rxd32[i].addr, 0xdeadbeef);
1096
1097	di->rxin = NEXTRXD(i);
1098
1099	return (rxp);
1100	}
1101
1102	static void
1103	dma32_txrotate(di_t *di)
1104	{
1105	uint ad;
1106	uint nactive;
1107	uint rot;
1108	uint old, new;
1109	uint32 w;
1110	uint first, last;
1111
1112	ASSERT(dma_txsuspendedidle(di));
1113
1114	nactive = dma_txactive(di);
1115	ad = B2I(((R_REG(&di->d32txregs->status) & XS_AD_MASK) >> XS_AD_SHIFT), dma32dd_t);
1116	rot = TXD(ad - di->txin);
1117
1118	ASSERT(rot < di->ntxd);
1119
1120	/* full-ring case is a lot harder - don't worry about this */
1121	if (rot >= (di->ntxd - nactive)) {
1122	DMA_ERROR(("%s: dma_txrotate: ring full - punt\n", di->name));
1123	return;
1124	}
1125
1126	first = di->txin;
1127	last = PREVTXD(di->txout);
1128
1129	/* move entries starting at last and moving backwards to first */
1130	for (old = last; old != PREVTXD(first); old = PREVTXD(old)) {
1131	new = TXD(old + rot);
1132
1133	/*
1134	* Move the tx dma descriptor.
1135	* EOT is set only in the last entry in the ring.
1136	*/
1137	w = R_SM(&di->txd32[old].ctrl) & ~CTRL_EOT;
1138	if (new == (di->ntxd - 1))
1139	w \|= CTRL_EOT;
1140	W_SM(&di->txd32[new].ctrl, w);
1141	W_SM(&di->txd32[new].addr, R_SM(&di->txd32[old].addr));
1142
1143	/* zap the old tx dma descriptor address field */
1144	W_SM(&di->txd32[old].addr, 0xdeadbeef);
1145
1146	/* move the corresponding txp[] entry */
1147	ASSERT(di->txp[new] == NULL);
1148	di->txp[new] = di->txp[old];
1149	di->txp[old] = NULL;
1150	}
1151
1152	/* update txin and txout */
1153	di->txin = ad;
1154	di->txout = TXD(di->txout + rot);
1155	di->txavail = di->ntxd - NTXDACTIVE(di->txin, di->txout) - 1;
1156
1157	/* kick the chip */
1158	W_REG(&di->d32txregs->ptr, I2B(di->txout, dma32dd_t));
1159	}
1160
1161	/* 64 bits DMA non-inline functions */
1162
1163	#ifdef BCMDMA64
1164
1165	static bool
1166	dma64_alloc(dma_info_t *di, uint direction)
1167	{
1168	uint size;
1169	uint ddlen;
1170	uint32 alignbytes;
1171	void *va;
1172
1173	ddlen = sizeof (dma64dd_t);
1174
1175	size = (direction == DMA_TX) ? (di->ntxd * ddlen) : (di->nrxd * ddlen);
1176
1177	alignbytes = di->dma64align;
1178
1179	if (!ISALIGNED(DMA_CONSISTENT_ALIGN, alignbytes))
1180	size += alignbytes;
1181
1182
1183	if (direction == DMA_TX) {
1184	if ((va = DMA_ALLOC_CONSISTENT(di->osh, size, &di->txdpa)) == NULL) {
1185	DMA_ERROR(("%s: dma_attach: DMA_ALLOC_CONSISTENT(ntxd) failed\n", di->name));
1186	return FALSE;
1187	}
1188
1189	di->txd64 = (dma64dd_t*) ROUNDUP((uintptr)va, alignbytes);
1190	di->txdalign = (uint)((int8)di->txd64 - (int8)va);
1191	di->txdpa += di->txdalign;
1192	di->txdalloc = size;
1193	ASSERT(ISALIGNED((uintptr)di->txd64, alignbytes));
1194	} else {
1195	if ((va = DMA_ALLOC_CONSISTENT(di->osh, size, &di->rxdpa)) == NULL) {
1196	DMA_ERROR(("%s: dma_attach: DMA_ALLOC_CONSISTENT(nrxd) failed\n", di->name));
1197	return FALSE;
1198	}
1199	di->rxd64 = (dma64dd_t*) ROUNDUP((uintptr)va, alignbytes);
1200	di->rxdalign = (uint)((int8)di->rxd64 - (int8)va);
1201	di->rxdpa += di->rxdalign;
1202	di->rxdalloc = size;
1203	ASSERT(ISALIGNED((uintptr)di->rxd64, alignbytes));
1204	}
1205
1206	return TRUE;
1207	}
1208
1209	static void
1210	dma64_txreset(dma_info_t *di)
1211	{
1212	uint32 status;
1213
1214	/* suspend tx DMA first */
1215	W_REG(&di->d64txregs->control, D64_XC_SE);
1216	SPINWAIT((status = (R_REG(&di->d64txregs->status0) & D64_XS0_XS_MASK)) != D64_XS0_XS_DISABLED &&
1217	status != D64_XS0_XS_IDLE &&
1218	status != D64_XS0_XS_STOPPED,
1219	10000);
1220
1221	W_REG(&di->d64txregs->control, 0);
1222	SPINWAIT((status = (R_REG(&di->d64txregs->status0) & D64_XS0_XS_MASK)) != D64_XS0_XS_DISABLED,
1223	10000);
1224
1225	if (status != D64_XS0_XS_DISABLED) {
1226	DMA_ERROR(("%s: dma_txreset: dma cannot be stopped\n", di->name));
1227	}
1228
1229	/* wait for the last transaction to complete */
1230	OSL_DELAY(300);
1231	}
1232
1233	static void
1234	dma64_rxreset(dma_info_t *di)
1235	{
1236	uint32 status;
1237
1238	W_REG(&di->d64rxregs->control, 0);
1239	SPINWAIT((status = (R_REG(&di->d64rxregs->status0) & D64_RS0_RS_MASK)) != D64_RS0_RS_DISABLED,
1240	10000);
1241
1242	if (status != D64_RS0_RS_DISABLED) {
1243	DMA_ERROR(("%s: dma_rxreset: dma cannot be stopped\n", di->name));
1244	}
1245	}
1246
1247	static bool
1248	dma64_txsuspendedidle(dma_info_t *di)
1249	{
1250
1251	if (!(R_REG(&di->d64txregs->control) & D64_XC_SE))
1252	return 0;
1253
1254	if ((R_REG(&di->d64txregs->status0) & D64_XS0_XS_MASK) == D64_XS0_XS_IDLE)
1255	return 1;
1256
1257	return 0;
1258	}
1259
1260	/*
1261	* supports full 32bit dma engine buffer addressing so
1262	* dma buffers can cross 4 Kbyte page boundaries.
1263	*/
1264	static int
1265	dma64_txfast(dma_info_t di, void p0, uint32 coreflags)
1266	{
1267	void p, next;
1268	uchar *data;
1269	uint len;
1270	uint txout;
1271	uint32 flags;
1272	uint32 pa;
1273
1274	DMA_TRACE(("%s: dma_txfast\n", di->name));
1275
1276	txout = di->txout;
1277	flags = 0;
1278
1279	/*
1280	* Walk the chain of packet buffers
1281	* allocating and initializing transmit descriptor entries.
1282	*/
1283	for (p = p0; p; p = next) {
1284	data = PKTDATA(di->osh, p);
1285	len = PKTLEN(di->osh, p);
1286	next = PKTNEXT(di->osh, p);
1287
1288	/* return nonzero if out of tx descriptors */
1289	if (NEXTTXD(txout) == di->txin)
1290	goto outoftxd;
1291
1292	if (len == 0)
1293	continue;
1294
1295	/* get physical address of buffer start */
1296	pa = (uint32) DMA_MAP(di->osh, data, len, DMA_TX, p);
1297
1298	flags = coreflags;
1299
1300	if (p == p0)
1301	flags \|= D64_CTRL1_SOF;
1302	if (next == NULL)
1303	flags \|= (D64_CTRL1_IOC \| D64_CTRL1_EOF);
1304	if (txout == (di->ntxd - 1))
1305	flags \|= D64_CTRL1_EOT;
1306
1307	dma64_dd_upd(di, di->txd64, pa, txout, &flags, len);
1308
1309	ASSERT(di->txp[txout] == NULL);
1310
1311	txout = NEXTTXD(txout);
1312	}
1313
1314	/* if last txd eof not set, fix it */
1315	if (!(flags & D64_CTRL1_EOF))
1316	W_SM(&di->txd64[PREVTXD(txout)].ctrl1, BUS_SWAP32(flags \| D64_CTRL1_IOC \| D64_CTRL1_EOF));
1317
1318	/* save the packet */
1319	di->txp[PREVTXD(txout)] = p0;
1320
1321	/* bump the tx descriptor index */
1322	di->txout = txout;
1323
1324	/* kick the chip */
1325	W_REG(&di->d64txregs->ptr, I2B(txout, dma64dd_t));
1326
1327	/* tx flow control */
1328	di->txavail = di->ntxd - NTXDACTIVE(di->txin, di->txout) - 1;
1329
1330	return (0);
1331
1332	outoftxd:
1333	DMA_ERROR(("%s: dma_txfast: out of txds\n", di->name));
1334	PKTFREE(di->osh, p0, TRUE);
1335	di->txavail = 0;
1336	di->hnddma.txnobuf++;
1337	return (-1);
1338	}
1339
1340	static void*
1341	dma64_getnexttxp(dma_info_t *di, bool forceall)
1342	{
1343	uint start, end, i;
1344	void *txp;
1345
1346	DMA_TRACE(("%s: dma_getnexttxp %s\n", di->name, forceall ? "all" : ""));
1347
1348	txp = NULL;
1349
1350	start = di->txin;
1351	if (forceall)
1352	end = di->txout;
1353	else
1354	end = B2I(R_REG(&di->d64txregs->status0) & D64_XS0_CD_MASK, dma64dd_t);
1355
1356	if ((start == 0) && (end > di->txout))
1357	goto bogus;
1358
1359	for (i = start; i != end && !txp; i = NEXTTXD(i)) {
1360	DMA_UNMAP(di->osh, (BUS_SWAP32(R_SM(&di->txd64[i].addrlow)) - di->dataoffsetlow),
1361	(BUS_SWAP32(R_SM(&di->txd64[i].ctrl2)) & D64_CTRL2_BC_MASK), DMA_TX, di->txp[i]);
1362
1363	W_SM(&di->txd64[i].addrlow, 0xdeadbeef);
1364	W_SM(&di->txd64[i].addrhigh, 0xdeadbeef);
1365
1366	txp = di->txp[i];
1367	di->txp[i] = NULL;
1368	}
1369
1370	di->txin = i;
1371
1372	/* tx flow control */
1373	di->txavail = di->ntxd - NTXDACTIVE(di->txin, di->txout) - 1;
1374
1375	return (txp);
1376
1377	bogus:
1378	/*
1379	DMA_ERROR(("dma_getnexttxp: bogus curr: start %d end %d txout %d force %d\n",
1380	start, end, di->txout, forceall));
1381	*/
1382	return (NULL);
1383	}
1384
1385	static void *
1386	dma64_getnextrxp(dma_info_t *di, bool forceall)
1387	{
1388	uint i;
1389	void *rxp;
1390
1391	/* if forcing, dma engine must be disabled */
1392	ASSERT(!forceall \|\| !dma_rxenabled(di));
1393
1394	i = di->rxin;
1395
1396	/* return if no packets posted */
1397	if (i == di->rxout)
1398	return (NULL);
1399
1400	/* ignore curr if forceall */
1401	if (!forceall && (i == B2I(R_REG(&di->d64rxregs->status0) & D64_RS0_CD_MASK, dma64dd_t)))
1402	return (NULL);
1403
1404	/* get the packet pointer that corresponds to the rx descriptor */
1405	rxp = di->rxp[i];
1406	ASSERT(rxp);
1407	di->rxp[i] = NULL;
1408
1409	/* clear this packet from the descriptor ring */
1410	DMA_UNMAP(di->osh, (BUS_SWAP32(R_SM(&di->rxd64[i].addrlow)) - di->dataoffsetlow),
1411	di->rxbufsize, DMA_RX, rxp);
1412
1413	W_SM(&di->rxd64[i].addrlow, 0xdeadbeef);
1414	W_SM(&di->rxd64[i].addrhigh, 0xdeadbeef);
1415
1416	di->rxin = NEXTRXD(i);
1417
1418	return (rxp);
1419	}
1420
1421	static void
1422	dma64_txrotate(di_t *di)
1423	{
1424	uint ad;
1425	uint nactive;
1426	uint rot;
1427	uint old, new;
1428	uint32 w;
1429	uint first, last;
1430
1431	ASSERT(dma_txsuspendedidle(di));
1432
1433	nactive = dma_txactive(di);
1434	ad = B2I((R_REG(&di->d64txregs->status1) & D64_XS1_AD_MASK), dma64dd_t);
1435	rot = TXD(ad - di->txin);
1436
1437	ASSERT(rot < di->ntxd);
1438
1439	/* full-ring case is a lot harder - don't worry about this */
1440	if (rot >= (di->ntxd - nactive)) {
1441	DMA_ERROR(("%s: dma_txrotate: ring full - punt\n", di->name));
1442	return;
1443	}
1444
1445	first = di->txin;
1446	last = PREVTXD(di->txout);
1447
1448	/* move entries starting at last and moving backwards to first */
1449	for (old = last; old != PREVTXD(first); old = PREVTXD(old)) {
1450	new = TXD(old + rot);
1451
1452	/*
1453	* Move the tx dma descriptor.
1454	* EOT is set only in the last entry in the ring.
1455	*/
1456	w = R_SM(&di->txd64[old].ctrl1) & ~D64_CTRL1_EOT;
1457	if (new == (di->ntxd - 1))
1458	w \|= D64_CTRL1_EOT;
1459	W_SM(&di->txd64[new].ctrl1, w);
1460
1461	w = R_SM(&di->txd64[old].ctrl2);
1462	W_SM(&di->txd64[new].ctrl2, w);
1463
1464	W_SM(&di->txd64[new].addrlow, R_SM(&di->txd64[old].addrlow));
1465	W_SM(&di->txd64[new].addrhigh, R_SM(&di->txd64[old].addrhigh));
1466
1467	/* zap the old tx dma descriptor address field */
1468	W_SM(&di->txd64[old].addrlow, 0xdeadbeef);
1469	W_SM(&di->txd64[old].addrhigh, 0xdeadbeef);
1470
1471	/* move the corresponding txp[] entry */
1472	ASSERT(di->txp[new] == NULL);
1473	di->txp[new] = di->txp[old];
1474	di->txp[old] = NULL;
1475	}
1476
1477	/* update txin and txout */
1478	di->txin = ad;
1479	di->txout = TXD(di->txout + rot);
1480	di->txavail = di->ntxd - NTXDACTIVE(di->txin, di->txout) - 1;
1481
1482	/* kick the chip */
1483	W_REG(&di->d64txregs->ptr, I2B(di->txout, dma64dd_t));
1484	}
1485
1486	#endif
1487

Note: See TracBrowser for help on using the browser.

root/src/shared/hnddma.c

Download in other formats: