1/*-
2 * SPDX-License-Identifier: BSD-3-Clause AND BSD-2-Clause
3 *
4 * Copyright (c) KATO Takenori, 1999.
5 *
6 * All rights reserved. Unpublished rights reserved under the copyright
7 * laws of Japan.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 *
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer as
15 * the first lines of this file unmodified.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
19 * 3. The name of the author may not be used to endorse or promote products
20 * derived from this software without specific prior written permission.
21 *
22 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
23 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
24 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
25 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
26 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
27 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
31 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 */
33
34/* $NetBSD: bus.h,v 1.12 1997/10/01 08:25:15 fvdl Exp $ */
35
36/*-
37 * Copyright (c) 1996, 1997 The NetBSD Foundation, Inc.
38 * All rights reserved.
39 *
40 * This code is derived from software contributed to The NetBSD Foundation
41 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
42 * NASA Ames Research Center.
43 *
44 * Redistribution and use in source and binary forms, with or without
45 * modification, are permitted provided that the following conditions
46 * are met:
47 * 1. Redistributions of source code must retain the above copyright
48 * notice, this list of conditions and the following disclaimer.
49 * 2. Redistributions in binary form must reproduce the above copyright
50 * notice, this list of conditions and the following disclaimer in the
51 * documentation and/or other materials provided with the distribution.
52 *
53 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
54 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
55 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
56 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
57 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
58 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
59 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
60 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
61 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
62 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
63 * POSSIBILITY OF SUCH DAMAGE.
64 */
65
66/*-
67 * Copyright (c) 1996 Charles M. Hannum. All rights reserved.
68 * Copyright (c) 1996 Christopher G. Demetriou. All rights reserved.
69 *
70 * Redistribution and use in source and binary forms, with or without
71 * modification, are permitted provided that the following conditions
72 * are met:
73 * 1. Redistributions of source code must retain the above copyright
74 * notice, this list of conditions and the following disclaimer.
75 * 2. Redistributions in binary form must reproduce the above copyright
76 * notice, this list of conditions and the following disclaimer in the
77 * documentation and/or other materials provided with the distribution.
78 * 3. All advertising materials mentioning features or use of this software
79 * must display the following acknowledgement:
80 * This product includes software developed by Christopher G. Demetriou
81 * for the NetBSD Project.
82 * 4. The name of the author may not be used to endorse or promote products
83 * derived from this software without specific prior written permission
84 *
85 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
86 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
87 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
88 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
89 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
90 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
91 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
92 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
93 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
94 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
95 */
96
97#ifndef _MACHINE_BUS_H_
98#define _MACHINE_BUS_H_
99
100#include <machine/_bus.h>
101#include <machine/cpufunc.h>
102#include <machine/bus_dma.h>
103
104/*
105 * Values for the x86 bus space tag, not to be used directly by MI code.
106 */
107#define X86_BUS_SPACE_IO 0 /* space is i/o space */
108#define X86_BUS_SPACE_MEM 1 /* space is mem space */
109
110#define BUS_SPACE_MAXSIZE_24BIT 0xFFFFFF
111#define BUS_SPACE_MAXSIZE_32BIT 0xFFFFFFFF
112#if defined(__amd64__)
113#define BUS_SPACE_MAXSIZE 0xFFFFFFFFFFFFFFFFULL
114#else
115#define BUS_SPACE_MAXSIZE 0xFFFFFFFF
116#endif
117#define BUS_SPACE_MAXADDR_24BIT 0xFFFFFF
118#define BUS_SPACE_MAXADDR_32BIT 0xFFFFFFFF
119#if defined(__amd64__) || defined(PAE)
120#define BUS_SPACE_MAXADDR_48BIT 0xFFFFFFFFFFFFULL
121#define BUS_SPACE_MAXADDR 0xFFFFFFFFFFFFFFFFULL
122#else
123#define BUS_SPACE_MAXADDR 0xFFFFFFFF
124#endif
125
126#define BUS_SPACE_INVALID_DATA (~0)
127#define BUS_SPACE_UNRESTRICTED (~0)
128
129#define BUS_SPACE_BARRIER_READ 0x01 /* force read barrier */
130#define BUS_SPACE_BARRIER_WRITE 0x02 /* force write barrier */
131
132#if defined(SAN_NEEDS_INTERCEPTORS) && !defined(SAN_RUNTIME)
133#include <sys/bus_san.h>
134#else
135
136/*
137 * Map a region of device bus space into CPU virtual address space.
138 */
139
140int bus_space_map(bus_space_tag_t tag, bus_addr_t addr, bus_size_t size,
141 int flags, bus_space_handle_t *bshp);
142
143/*
144 * Unmap a region of device bus space.
145 */
146
147void bus_space_unmap(bus_space_tag_t tag, bus_space_handle_t bsh,
148 bus_size_t size);
149
150/*
151 * Get a new handle for a subregion of an already-mapped area of bus space.
152 */
153
154static __inline int bus_space_subregion(bus_space_tag_t t,
155 bus_space_handle_t bsh,
156 bus_size_t offset, bus_size_t size,
157 bus_space_handle_t *nbshp);
158
159static __inline int
160bus_space_subregion(bus_space_tag_t t __unused, bus_space_handle_t bsh,
161 bus_size_t offset, bus_size_t size __unused,
162 bus_space_handle_t *nbshp)
163{
164
165 *nbshp = bsh + offset;
166 return (0);
167}
168
169/*
170 * Allocate a region of memory that is accessible to devices in bus space.
171 */
172
173int bus_space_alloc(bus_space_tag_t t, bus_addr_t rstart,
174 bus_addr_t rend, bus_size_t size, bus_size_t align,
175 bus_size_t boundary, int flags, bus_addr_t *addrp,
176 bus_space_handle_t *bshp);
177
178/*
179 * Free a region of bus space accessible memory.
180 */
181
182static __inline void bus_space_free(bus_space_tag_t t, bus_space_handle_t bsh,
183 bus_size_t size);
184
185static __inline void
186bus_space_free(bus_space_tag_t t __unused, bus_space_handle_t bsh __unused,
187 bus_size_t size __unused)
188{
189}
190
191/*
192 * Read a 1, 2, 4, or 8 byte quantity from bus space
193 * described by tag/handle/offset.
194 */
195static __inline u_int8_t bus_space_read_1(bus_space_tag_t tag,
196 bus_space_handle_t handle,
197 bus_size_t offset);
198
199static __inline u_int16_t bus_space_read_2(bus_space_tag_t tag,
200 bus_space_handle_t handle,
201 bus_size_t offset);
202
203static __inline u_int32_t bus_space_read_4(bus_space_tag_t tag,
204 bus_space_handle_t handle,
205 bus_size_t offset);
206
207#ifdef __amd64__
208static __inline uint64_t bus_space_read_8(bus_space_tag_t tag,
209 bus_space_handle_t handle,
210 bus_size_t offset);
211#endif
212
213static __inline u_int8_t
214bus_space_read_1(bus_space_tag_t tag, bus_space_handle_t handle,
215 bus_size_t offset)
216{
217
218 if (tag == X86_BUS_SPACE_IO)
219 return (inb(handle + offset));
220 return (*(volatile u_int8_t *)(handle + offset));
221}
222
223static __inline u_int16_t
224bus_space_read_2(bus_space_tag_t tag, bus_space_handle_t handle,
225 bus_size_t offset)
226{
227
228 if (tag == X86_BUS_SPACE_IO)
229 return (inw(handle + offset));
230 return (*(volatile u_int16_t *)(handle + offset));
231}
232
233static __inline u_int32_t
234bus_space_read_4(bus_space_tag_t tag, bus_space_handle_t handle,
235 bus_size_t offset)
236{
237
238 if (tag == X86_BUS_SPACE_IO)
239 return (inl(handle + offset));
240 return (*(volatile u_int32_t *)(handle + offset));
241}
242
243#ifdef __amd64__
244static __inline uint64_t
245bus_space_read_8(bus_space_tag_t tag, bus_space_handle_t handle,
246 bus_size_t offset)
247{
248
249 if (tag == X86_BUS_SPACE_IO) /* No 8 byte IO space access on x86 */
250 return (BUS_SPACE_INVALID_DATA);
251 return (*(volatile uint64_t *)(handle + offset));
252}
253#endif
254
255/*
256 * Read `count' 1, 2, 4, or 8 byte quantities from bus space
257 * described by tag/handle/offset and copy into buffer provided.
258 */
259static __inline void bus_space_read_multi_1(bus_space_tag_t tag,
260 bus_space_handle_t bsh,
261 bus_size_t offset, u_int8_t *addr,
262 size_t count);
263
264static __inline void bus_space_read_multi_2(bus_space_tag_t tag,
265 bus_space_handle_t bsh,
266 bus_size_t offset, u_int16_t *addr,
267 size_t count);
268
269static __inline void bus_space_read_multi_4(bus_space_tag_t tag,
270 bus_space_handle_t bsh,
271 bus_size_t offset, u_int32_t *addr,
272 size_t count);
273
274static __inline void
275bus_space_read_multi_1(bus_space_tag_t tag, bus_space_handle_t bsh,
276 bus_size_t offset, u_int8_t *addr, size_t count)
277{
278
279 if (tag == X86_BUS_SPACE_IO)
280 insb(bsh + offset, addr, count);
281 else {
282 __asm __volatile(" \n\
283 1: movb (%2),%%al \n\
284 stosb \n\
285 loop 1b" :
286 "=D" (addr), "=c" (count) :
287 "r" (bsh + offset), "0" (addr), "1" (count) :
288 "%eax", "memory");
289 }
290}
291
292static __inline void
293bus_space_read_multi_2(bus_space_tag_t tag, bus_space_handle_t bsh,
294 bus_size_t offset, u_int16_t *addr, size_t count)
295{
296
297 if (tag == X86_BUS_SPACE_IO)
298 insw(bsh + offset, addr, count);
299 else {
300 __asm __volatile(" \n\
301 1: movw (%2),%%ax \n\
302 stosw \n\
303 loop 1b" :
304 "=D" (addr), "=c" (count) :
305 "r" (bsh + offset), "0" (addr), "1" (count) :
306 "%eax", "memory");
307 }
308}
309
310static __inline void
311bus_space_read_multi_4(bus_space_tag_t tag, bus_space_handle_t bsh,
312 bus_size_t offset, u_int32_t *addr, size_t count)
313{
314
315 if (tag == X86_BUS_SPACE_IO)
316 insl(bsh + offset, addr, count);
317 else {
318 __asm __volatile(" \n\
319 1: movl (%2),%%eax \n\
320 stosl \n\
321 loop 1b" :
322 "=D" (addr), "=c" (count) :
323 "r" (bsh + offset), "0" (addr), "1" (count) :
324 "%eax", "memory");
325 }
326}
327
328#if 0 /* Cause a link error for bus_space_read_multi_8 */
329#define bus_space_read_multi_8 !!! bus_space_read_multi_8 unimplemented !!!
330#endif
331
332/*
333 * Read `count' 1, 2, 4, or 8 byte quantities from bus space
334 * described by tag/handle and starting at `offset' and copy into
335 * buffer provided.
336 */
337static __inline void bus_space_read_region_1(bus_space_tag_t tag,
338 bus_space_handle_t bsh,
339 bus_size_t offset, u_int8_t *addr,
340 size_t count);
341
342static __inline void bus_space_read_region_2(bus_space_tag_t tag,
343 bus_space_handle_t bsh,
344 bus_size_t offset, u_int16_t *addr,
345 size_t count);
346
347static __inline void bus_space_read_region_4(bus_space_tag_t tag,
348 bus_space_handle_t bsh,
349 bus_size_t offset, u_int32_t *addr,
350 size_t count);
351
352static __inline void
353bus_space_read_region_1(bus_space_tag_t tag, bus_space_handle_t bsh,
354 bus_size_t offset, u_int8_t *addr, size_t count)
355{
356
357 if (tag == X86_BUS_SPACE_IO) {
358 int _port_ = bsh + offset;
359 __asm __volatile(" \n\
360 1: inb %w2,%%al \n\
361 stosb \n\
362 incl %2 \n\
363 loop 1b" :
364 "=D" (addr), "=c" (count), "=d" (_port_) :
365 "0" (addr), "1" (count), "2" (_port_) :
366 "%eax", "memory", "cc");
367 } else {
368 bus_space_handle_t _port_ = bsh + offset;
369 __asm __volatile(" \n\
370 repne \n\
371 movsb" :
372 "=D" (addr), "=c" (count), "=S" (_port_) :
373 "0" (addr), "1" (count), "2" (_port_) :
374 "memory", "cc");
375 }
376}
377
378static __inline void
379bus_space_read_region_2(bus_space_tag_t tag, bus_space_handle_t bsh,
380 bus_size_t offset, u_int16_t *addr, size_t count)
381{
382
383 if (tag == X86_BUS_SPACE_IO) {
384 int _port_ = bsh + offset;
385 __asm __volatile(" \n\
386 1: inw %w2,%%ax \n\
387 stosw \n\
388 addl $2,%2 \n\
389 loop 1b" :
390 "=D" (addr), "=c" (count), "=d" (_port_) :
391 "0" (addr), "1" (count), "2" (_port_) :
392 "%eax", "memory", "cc");
393 } else {
394 bus_space_handle_t _port_ = bsh + offset;
395 __asm __volatile(" \n\
396 repne \n\
397 movsw" :
398 "=D" (addr), "=c" (count), "=S" (_port_) :
399 "0" (addr), "1" (count), "2" (_port_) :
400 "memory", "cc");
401 }
402}
403
404static __inline void
405bus_space_read_region_4(bus_space_tag_t tag, bus_space_handle_t bsh,
406 bus_size_t offset, u_int32_t *addr, size_t count)
407{
408
409 if (tag == X86_BUS_SPACE_IO) {
410 int _port_ = bsh + offset;
411 __asm __volatile(" \n\
412 1: inl %w2,%%eax \n\
413 stosl \n\
414 addl $4,%2 \n\
415 loop 1b" :
416 "=D" (addr), "=c" (count), "=d" (_port_) :
417 "0" (addr), "1" (count), "2" (_port_) :
418 "%eax", "memory", "cc");
419 } else {
420 bus_space_handle_t _port_ = bsh + offset;
421 __asm __volatile(" \n\
422 repne \n\
423 movsl" :
424 "=D" (addr), "=c" (count), "=S" (_port_) :
425 "0" (addr), "1" (count), "2" (_port_) :
426 "memory", "cc");
427 }
428}
429
430#if 0 /* Cause a link error for bus_space_read_region_8 */
431#define bus_space_read_region_8 !!! bus_space_read_region_8 unimplemented !!!
432#endif
433
434/*
435 * Write the 1, 2, 4, or 8 byte value `value' to bus space
436 * described by tag/handle/offset.
437 */
438
439static __inline void bus_space_write_1(bus_space_tag_t tag,
440 bus_space_handle_t bsh,
441 bus_size_t offset, u_int8_t value);
442
443static __inline void bus_space_write_2(bus_space_tag_t tag,
444 bus_space_handle_t bsh,
445 bus_size_t offset, u_int16_t value);
446
447static __inline void bus_space_write_4(bus_space_tag_t tag,
448 bus_space_handle_t bsh,
449 bus_size_t offset, u_int32_t value);
450
451#ifdef __amd64__
452static __inline void bus_space_write_8(bus_space_tag_t tag,
453 bus_space_handle_t bsh,
454 bus_size_t offset, uint64_t value);
455#endif
456
457static __inline void
458bus_space_write_1(bus_space_tag_t tag, bus_space_handle_t bsh,
459 bus_size_t offset, u_int8_t value)
460{
461
462 if (tag == X86_BUS_SPACE_IO)
463 outb(bsh + offset, value);
464 else
465 *(volatile u_int8_t *)(bsh + offset) = value;
466}
467
468static __inline void
469bus_space_write_2(bus_space_tag_t tag, bus_space_handle_t bsh,
470 bus_size_t offset, u_int16_t value)
471{
472
473 if (tag == X86_BUS_SPACE_IO)
474 outw(bsh + offset, value);
475 else
476 *(volatile u_int16_t *)(bsh + offset) = value;
477}
478
479static __inline void
480bus_space_write_4(bus_space_tag_t tag, bus_space_handle_t bsh,
481 bus_size_t offset, u_int32_t value)
482{
483
484 if (tag == X86_BUS_SPACE_IO)
485 outl(bsh + offset, value);
486 else
487 *(volatile u_int32_t *)(bsh + offset) = value;
488}
489
490#ifdef __amd64__
491static __inline void
492bus_space_write_8(bus_space_tag_t tag, bus_space_handle_t bsh,
493 bus_size_t offset, uint64_t value)
494{
495
496 if (tag == X86_BUS_SPACE_IO) /* No 8 byte IO space access on x86 */
497 return;
498 else
499 *(volatile uint64_t *)(bsh + offset) = value;
500}
501#endif
502
503/*
504 * Write `count' 1, 2, 4, or 8 byte quantities from the buffer
505 * provided to bus space described by tag/handle/offset.
506 */
507
508static __inline void bus_space_write_multi_1(bus_space_tag_t tag,
509 bus_space_handle_t bsh,
510 bus_size_t offset,
511 const u_int8_t *addr,
512 size_t count);
513static __inline void bus_space_write_multi_2(bus_space_tag_t tag,
514 bus_space_handle_t bsh,
515 bus_size_t offset,
516 const u_int16_t *addr,
517 size_t count);
518
519static __inline void bus_space_write_multi_4(bus_space_tag_t tag,
520 bus_space_handle_t bsh,
521 bus_size_t offset,
522 const u_int32_t *addr,
523 size_t count);
524
525static __inline void
526bus_space_write_multi_1(bus_space_tag_t tag, bus_space_handle_t bsh,
527 bus_size_t offset, const u_int8_t *addr, size_t count)
528{
529
530 if (tag == X86_BUS_SPACE_IO)
531 outsb(bsh + offset, addr, count);
532 else {
533 __asm __volatile(" \n\
534 1: lodsb \n\
535 movb %%al,(%2) \n\
536 loop 1b" :
537 "=S" (addr), "=c" (count) :
538 "r" (bsh + offset), "0" (addr), "1" (count) :
539 "%eax", "memory", "cc");
540 }
541}
542
543static __inline void
544bus_space_write_multi_2(bus_space_tag_t tag, bus_space_handle_t bsh,
545 bus_size_t offset, const u_int16_t *addr, size_t count)
546{
547
548 if (tag == X86_BUS_SPACE_IO)
549 outsw(bsh + offset, addr, count);
550 else {
551 __asm __volatile(" \n\
552 1: lodsw \n\
553 movw %%ax,(%2) \n\
554 loop 1b" :
555 "=S" (addr), "=c" (count) :
556 "r" (bsh + offset), "0" (addr), "1" (count) :
557 "%eax", "memory", "cc");
558 }
559}
560
561static __inline void
562bus_space_write_multi_4(bus_space_tag_t tag, bus_space_handle_t bsh,
563 bus_size_t offset, const u_int32_t *addr, size_t count)
564{
565
566 if (tag == X86_BUS_SPACE_IO)
567 outsl(bsh + offset, addr, count);
568 else {
569 __asm __volatile(" \n\
570 1: lodsl \n\
571 movl %%eax,(%2) \n\
572 loop 1b" :
573 "=S" (addr), "=c" (count) :
574 "r" (bsh + offset), "0" (addr), "1" (count) :
575 "%eax", "memory", "cc");
576 }
577}
578
579#if 0 /* Cause a link error for bus_space_write_multi_8 */
580#define bus_space_write_multi_8(t, h, o, a, c) \
581 !!! bus_space_write_multi_8 unimplemented !!!
582#endif
583
584/*
585 * Write `count' 1, 2, 4, or 8 byte quantities from the buffer provided
586 * to bus space described by tag/handle starting at `offset'.
587 */
588
589static __inline void bus_space_write_region_1(bus_space_tag_t tag,
590 bus_space_handle_t bsh,
591 bus_size_t offset,
592 const u_int8_t *addr,
593 size_t count);
594static __inline void bus_space_write_region_2(bus_space_tag_t tag,
595 bus_space_handle_t bsh,
596 bus_size_t offset,
597 const u_int16_t *addr,
598 size_t count);
599static __inline void bus_space_write_region_4(bus_space_tag_t tag,
600 bus_space_handle_t bsh,
601 bus_size_t offset,
602 const u_int32_t *addr,
603 size_t count);
604
605static __inline void
606bus_space_write_region_1(bus_space_tag_t tag, bus_space_handle_t bsh,
607 bus_size_t offset, const u_int8_t *addr, size_t count)
608{
609
610 if (tag == X86_BUS_SPACE_IO) {
611 int _port_ = bsh + offset;
612 __asm __volatile(" \n\
613 1: lodsb \n\
614 outb %%al,%w0 \n\
615 incl %0 \n\
616 loop 1b" :
617 "=d" (_port_), "=S" (addr), "=c" (count) :
618 "0" (_port_), "1" (addr), "2" (count) :
619 "%eax", "memory", "cc");
620 } else {
621 bus_space_handle_t _port_ = bsh + offset;
622 __asm __volatile(" \n\
623 repne \n\
624 movsb" :
625 "=D" (_port_), "=S" (addr), "=c" (count) :
626 "0" (_port_), "1" (addr), "2" (count) :
627 "memory", "cc");
628 }
629}
630
631static __inline void
632bus_space_write_region_2(bus_space_tag_t tag, bus_space_handle_t bsh,
633 bus_size_t offset, const u_int16_t *addr, size_t count)
634{
635
636 if (tag == X86_BUS_SPACE_IO) {
637 int _port_ = bsh + offset;
638 __asm __volatile(" \n\
639 1: lodsw \n\
640 outw %%ax,%w0 \n\
641 addl $2,%0 \n\
642 loop 1b" :
643 "=d" (_port_), "=S" (addr), "=c" (count) :
644 "0" (_port_), "1" (addr), "2" (count) :
645 "%eax", "memory", "cc");
646 } else {
647 bus_space_handle_t _port_ = bsh + offset;
648 __asm __volatile(" \n\
649 repne \n\
650 movsw" :
651 "=D" (_port_), "=S" (addr), "=c" (count) :
652 "0" (_port_), "1" (addr), "2" (count) :
653 "memory", "cc");
654 }
655}
656
657static __inline void
658bus_space_write_region_4(bus_space_tag_t tag, bus_space_handle_t bsh,
659 bus_size_t offset, const u_int32_t *addr, size_t count)
660{
661
662 if (tag == X86_BUS_SPACE_IO) {
663 int _port_ = bsh + offset;
664 __asm __volatile(" \n\
665 1: lodsl \n\
666 outl %%eax,%w0 \n\
667 addl $4,%0 \n\
668 loop 1b" :
669 "=d" (_port_), "=S" (addr), "=c" (count) :
670 "0" (_port_), "1" (addr), "2" (count) :
671 "%eax", "memory", "cc");
672 } else {
673 bus_space_handle_t _port_ = bsh + offset;
674 __asm __volatile(" \n\
675 repne \n\
676 movsl" :
677 "=D" (_port_), "=S" (addr), "=c" (count) :
678 "0" (_port_), "1" (addr), "2" (count) :
679 "memory", "cc");
680 }
681}
682
683#if 0 /* Cause a link error for bus_space_write_region_8 */
684#define bus_space_write_region_8 \
685 !!! bus_space_write_region_8 unimplemented !!!
686#endif
687
688/*
689 * Write the 1, 2, 4, or 8 byte value `val' to bus space described
690 * by tag/handle/offset `count' times.
691 */
692
693static __inline void bus_space_set_multi_1(bus_space_tag_t tag,
694 bus_space_handle_t bsh,
695 bus_size_t offset,
696 u_int8_t value, size_t count);
697static __inline void bus_space_set_multi_2(bus_space_tag_t tag,
698 bus_space_handle_t bsh,
699 bus_size_t offset,
700 u_int16_t value, size_t count);
701static __inline void bus_space_set_multi_4(bus_space_tag_t tag,
702 bus_space_handle_t bsh,
703 bus_size_t offset,
704 u_int32_t value, size_t count);
705
706static __inline void
707bus_space_set_multi_1(bus_space_tag_t tag, bus_space_handle_t bsh,
708 bus_size_t offset, u_int8_t value, size_t count)
709{
710 bus_space_handle_t addr = bsh + offset;
711
712 if (tag == X86_BUS_SPACE_IO)
713 while (count--)
714 outb(addr, value);
715 else
716 while (count--)
717 *(volatile u_int8_t *)(addr) = value;
718}
719
720static __inline void
721bus_space_set_multi_2(bus_space_tag_t tag, bus_space_handle_t bsh,
722 bus_size_t offset, u_int16_t value, size_t count)
723{
724 bus_space_handle_t addr = bsh + offset;
725
726 if (tag == X86_BUS_SPACE_IO)
727 while (count--)
728 outw(addr, value);
729 else
730 while (count--)
731 *(volatile u_int16_t *)(addr) = value;
732}
733
734static __inline void
735bus_space_set_multi_4(bus_space_tag_t tag, bus_space_handle_t bsh,
736 bus_size_t offset, u_int32_t value, size_t count)
737{
738 bus_space_handle_t addr = bsh + offset;
739
740 if (tag == X86_BUS_SPACE_IO)
741 while (count--)
742 outl(addr, value);
743 else
744 while (count--)
745 *(volatile u_int32_t *)(addr) = value;
746}
747
748#if 0 /* Cause a link error for bus_space_set_multi_8 */
749#define bus_space_set_multi_8 !!! bus_space_set_multi_8 unimplemented !!!
750#endif
751
752/*
753 * Write `count' 1, 2, 4, or 8 byte value `val' to bus space described
754 * by tag/handle starting at `offset'.
755 */
756
757static __inline void bus_space_set_region_1(bus_space_tag_t tag,
758 bus_space_handle_t bsh,
759 bus_size_t offset, u_int8_t value,
760 size_t count);
761static __inline void bus_space_set_region_2(bus_space_tag_t tag,
762 bus_space_handle_t bsh,
763 bus_size_t offset, u_int16_t value,
764 size_t count);
765static __inline void bus_space_set_region_4(bus_space_tag_t tag,
766 bus_space_handle_t bsh,
767 bus_size_t offset, u_int32_t value,
768 size_t count);
769
770static __inline void
771bus_space_set_region_1(bus_space_tag_t tag, bus_space_handle_t bsh,
772 bus_size_t offset, u_int8_t value, size_t count)
773{
774 bus_space_handle_t addr = bsh + offset;
775
776 if (tag == X86_BUS_SPACE_IO)
777 for (; count != 0; count--, addr++)
778 outb(addr, value);
779 else
780 for (; count != 0; count--, addr++)
781 *(volatile u_int8_t *)(addr) = value;
782}
783
784static __inline void
785bus_space_set_region_2(bus_space_tag_t tag, bus_space_handle_t bsh,
786 bus_size_t offset, u_int16_t value, size_t count)
787{
788 bus_space_handle_t addr = bsh + offset;
789
790 if (tag == X86_BUS_SPACE_IO)
791 for (; count != 0; count--, addr += 2)
792 outw(addr, value);
793 else
794 for (; count != 0; count--, addr += 2)
795 *(volatile u_int16_t *)(addr) = value;
796}
797
798static __inline void
799bus_space_set_region_4(bus_space_tag_t tag, bus_space_handle_t bsh,
800 bus_size_t offset, u_int32_t value, size_t count)
801{
802 bus_space_handle_t addr = bsh + offset;
803
804 if (tag == X86_BUS_SPACE_IO)
805 for (; count != 0; count--, addr += 4)
806 outl(addr, value);
807 else
808 for (; count != 0; count--, addr += 4)
809 *(volatile u_int32_t *)(addr) = value;
810}
811
812#if 0 /* Cause a link error for bus_space_set_region_8 */
813#define bus_space_set_region_8 !!! bus_space_set_region_8 unimplemented !!!
814#endif
815
816/*
817 * Copy `count' 1, 2, 4, or 8 byte values from bus space starting
818 * at tag/bsh1/off1 to bus space starting at tag/bsh2/off2.
819 */
820
821static __inline void bus_space_copy_region_1(bus_space_tag_t tag,
822 bus_space_handle_t bsh1,
823 bus_size_t off1,
824 bus_space_handle_t bsh2,
825 bus_size_t off2, size_t count);
826
827static __inline void bus_space_copy_region_2(bus_space_tag_t tag,
828 bus_space_handle_t bsh1,
829 bus_size_t off1,
830 bus_space_handle_t bsh2,
831 bus_size_t off2, size_t count);
832
833static __inline void bus_space_copy_region_4(bus_space_tag_t tag,
834 bus_space_handle_t bsh1,
835 bus_size_t off1,
836 bus_space_handle_t bsh2,
837 bus_size_t off2, size_t count);
838
839static __inline void
840bus_space_copy_region_1(bus_space_tag_t tag, bus_space_handle_t bsh1,
841 bus_size_t off1, bus_space_handle_t bsh2,
842 bus_size_t off2, size_t count)
843{
844 bus_space_handle_t addr1 = bsh1 + off1;
845 bus_space_handle_t addr2 = bsh2 + off2;
846
847 if (tag == X86_BUS_SPACE_IO) {
848 if (addr1 >= addr2) {
849 /* src after dest: copy forward */
850 for (; count != 0; count--, addr1++, addr2++)
851 outb(addr2, inb(addr1));
852 } else {
853 /* dest after src: copy backwards */
854 for (addr1 += (count - 1), addr2 += (count - 1);
855 count != 0; count--, addr1--, addr2--)
856 outb(addr2, inb(addr1));
857 }
858 } else {
859 if (addr1 >= addr2) {
860 /* src after dest: copy forward */
861 for (; count != 0; count--, addr1++, addr2++)
862 *(volatile u_int8_t *)(addr2) =
863 *(volatile u_int8_t *)(addr1);
864 } else {
865 /* dest after src: copy backwards */
866 for (addr1 += (count - 1), addr2 += (count - 1);
867 count != 0; count--, addr1--, addr2--)
868 *(volatile u_int8_t *)(addr2) =
869 *(volatile u_int8_t *)(addr1);
870 }
871 }
872}
873
874static __inline void
875bus_space_copy_region_2(bus_space_tag_t tag, bus_space_handle_t bsh1,
876 bus_size_t off1, bus_space_handle_t bsh2,
877 bus_size_t off2, size_t count)
878{
879 bus_space_handle_t addr1 = bsh1 + off1;
880 bus_space_handle_t addr2 = bsh2 + off2;
881
882 if (tag == X86_BUS_SPACE_IO) {
883 if (addr1 >= addr2) {
884 /* src after dest: copy forward */
885 for (; count != 0; count--, addr1 += 2, addr2 += 2)
886 outw(addr2, inw(addr1));
887 } else {
888 /* dest after src: copy backwards */
889 for (addr1 += 2 * (count - 1), addr2 += 2 * (count - 1);
890 count != 0; count--, addr1 -= 2, addr2 -= 2)
891 outw(addr2, inw(addr1));
892 }
893 } else {
894 if (addr1 >= addr2) {
895 /* src after dest: copy forward */
896 for (; count != 0; count--, addr1 += 2, addr2 += 2)
897 *(volatile u_int16_t *)(addr2) =
898 *(volatile u_int16_t *)(addr1);
899 } else {
900 /* dest after src: copy backwards */
901 for (addr1 += 2 * (count - 1), addr2 += 2 * (count - 1);
902 count != 0; count--, addr1 -= 2, addr2 -= 2)
903 *(volatile u_int16_t *)(addr2) =
904 *(volatile u_int16_t *)(addr1);
905 }
906 }
907}
908
909static __inline void
910bus_space_copy_region_4(bus_space_tag_t tag, bus_space_handle_t bsh1,
911 bus_size_t off1, bus_space_handle_t bsh2,
912 bus_size_t off2, size_t count)
913{
914 bus_space_handle_t addr1 = bsh1 + off1;
915 bus_space_handle_t addr2 = bsh2 + off2;
916
917 if (tag == X86_BUS_SPACE_IO) {
918 if (addr1 >= addr2) {
919 /* src after dest: copy forward */
920 for (; count != 0; count--, addr1 += 4, addr2 += 4)
921 outl(addr2, inl(addr1));
922 } else {
923 /* dest after src: copy backwards */
924 for (addr1 += 4 * (count - 1), addr2 += 4 * (count - 1);
925 count != 0; count--, addr1 -= 4, addr2 -= 4)
926 outl(addr2, inl(addr1));
927 }
928 } else {
929 if (addr1 >= addr2) {
930 /* src after dest: copy forward */
931 for (; count != 0; count--, addr1 += 4, addr2 += 4)
932 *(volatile u_int32_t *)(addr2) =
933 *(volatile u_int32_t *)(addr1);
934 } else {
935 /* dest after src: copy backwards */
936 for (addr1 += 4 * (count - 1), addr2 += 4 * (count - 1);
937 count != 0; count--, addr1 -= 4, addr2 -= 4)
938 *(volatile u_int32_t *)(addr2) =
939 *(volatile u_int32_t *)(addr1);
940 }
941 }
942}
943
944#if 0 /* Cause a link error for bus_space_copy_8 */
945#define bus_space_copy_region_8 !!! bus_space_copy_region_8 unimplemented !!!
946#endif
947
948/*
949 * Bus read/write barrier methods.
950 *
951 * void bus_space_barrier(bus_space_tag_t tag, bus_space_handle_t bsh,
952 * bus_size_t offset, bus_size_t len, int flags);
953 *
954 *
955 * Note that BUS_SPACE_BARRIER_WRITE doesn't do anything other than
956 * prevent reordering by the compiler; all Intel x86 processors currently
957 * retire operations outside the CPU in program order.
958 */
959static __inline void
960bus_space_barrier(bus_space_tag_t tag __unused, bus_space_handle_t bsh __unused,
961 bus_size_t offset __unused, bus_size_t len __unused, int flags)
962{
963 if (flags & BUS_SPACE_BARRIER_READ)
964#ifdef __amd64__
965 __asm __volatile("lock; addl $0,0(%%rsp)" : : : "memory");
966#else
967 __asm __volatile("lock; addl $0,0(%%esp)" : : : "memory");
968#endif
969 else
970 __compiler_membar();
971}
972
973#ifdef BUS_SPACE_NO_LEGACY
974#undef inb
975#undef outb
976#define inb(a) compiler_error
977#define inw(a) compiler_error
978#define inl(a) compiler_error
979#define outb(a, b) compiler_error
980#define outw(a, b) compiler_error
981#define outl(a, b) compiler_error
982#endif
983
984/*
985 * Stream accesses are the same as normal accesses on x86; there are no
986 * supported bus systems with an endianess different from the host one.
987 */
988#define bus_space_read_stream_1(t, h, o) bus_space_read_1((t), (h), (o))
989#define bus_space_read_stream_2(t, h, o) bus_space_read_2((t), (h), (o))
990#define bus_space_read_stream_4(t, h, o) bus_space_read_4((t), (h), (o))
991
992#define bus_space_read_multi_stream_1(t, h, o, a, c) \
993 bus_space_read_multi_1((t), (h), (o), (a), (c))
994#define bus_space_read_multi_stream_2(t, h, o, a, c) \
995 bus_space_read_multi_2((t), (h), (o), (a), (c))
996#define bus_space_read_multi_stream_4(t, h, o, a, c) \
997 bus_space_read_multi_4((t), (h), (o), (a), (c))
998
999#define bus_space_write_stream_1(t, h, o, v) \
1000 bus_space_write_1((t), (h), (o), (v))
1001#define bus_space_write_stream_2(t, h, o, v) \
1002 bus_space_write_2((t), (h), (o), (v))
1003#define bus_space_write_stream_4(t, h, o, v) \
1004 bus_space_write_4((t), (h), (o), (v))
1005
1006#define bus_space_write_multi_stream_1(t, h, o, a, c) \
1007 bus_space_write_multi_1((t), (h), (o), (a), (c))
1008#define bus_space_write_multi_stream_2(t, h, o, a, c) \
1009 bus_space_write_multi_2((t), (h), (o), (a), (c))
1010#define bus_space_write_multi_stream_4(t, h, o, a, c) \
1011 bus_space_write_multi_4((t), (h), (o), (a), (c))
1012
1013#define bus_space_set_multi_stream_1(t, h, o, v, c) \
1014 bus_space_set_multi_1((t), (h), (o), (v), (c))
1015#define bus_space_set_multi_stream_2(t, h, o, v, c) \
1016 bus_space_set_multi_2((t), (h), (o), (v), (c))
1017#define bus_space_set_multi_stream_4(t, h, o, v, c) \
1018 bus_space_set_multi_4((t), (h), (o), (v), (c))
1019
1020#define bus_space_read_region_stream_1(t, h, o, a, c) \
1021 bus_space_read_region_1((t), (h), (o), (a), (c))
1022#define bus_space_read_region_stream_2(t, h, o, a, c) \
1023 bus_space_read_region_2((t), (h), (o), (a), (c))
1024#define bus_space_read_region_stream_4(t, h, o, a, c) \
1025 bus_space_read_region_4((t), (h), (o), (a), (c))
1026
1027#define bus_space_write_region_stream_1(t, h, o, a, c) \
1028 bus_space_write_region_1((t), (h), (o), (a), (c))
1029#define bus_space_write_region_stream_2(t, h, o, a, c) \
1030 bus_space_write_region_2((t), (h), (o), (a), (c))
1031#define bus_space_write_region_stream_4(t, h, o, a, c) \
1032 bus_space_write_region_4((t), (h), (o), (a), (c))
1033
1034#define bus_space_set_region_stream_1(t, h, o, v, c) \
1035 bus_space_set_region_1((t), (h), (o), (v), (c))
1036#define bus_space_set_region_stream_2(t, h, o, v, c) \
1037 bus_space_set_region_2((t), (h), (o), (v), (c))
1038#define bus_space_set_region_stream_4(t, h, o, v, c) \
1039 bus_space_set_region_4((t), (h), (o), (v), (c))
1040
1041#define bus_space_copy_region_stream_1(t, h1, o1, h2, o2, c) \
1042 bus_space_copy_region_1((t), (h1), (o1), (h2), (o2), (c))
1043#define bus_space_copy_region_stream_2(t, h1, o1, h2, o2, c) \
1044 bus_space_copy_region_2((t), (h1), (o1), (h2), (o2), (c))
1045#define bus_space_copy_region_stream_4(t, h1, o1, h2, o2, c) \
1046 bus_space_copy_region_4((t), (h1), (o1), (h2), (o2), (c))
1047
1048#define BUS_PEEK_FUNC(width, type) \
1049 static inline int \
1050 bus_space_peek_##width(bus_space_tag_t tag, \
1051 bus_space_handle_t hnd, bus_size_t offset, type *value) \
1052 { \
1053 type tmp; \
1054 tmp = bus_space_read_##width(tag, hnd, offset); \
1055 *value = (type)tmp; \
1056 return (0); \
1057 }
1058BUS_PEEK_FUNC(1, uint8_t)
1059BUS_PEEK_FUNC(2, uint16_t)
1060BUS_PEEK_FUNC(4, uint32_t)
1061#ifdef __amd64__
1062BUS_PEEK_FUNC(8, uint64_t)
1063#endif
1064
1065#define BUS_POKE_FUNC(width, type) \
1066 static inline int \
1067 bus_space_poke_##width(bus_space_tag_t tag, \
1068 bus_space_handle_t hnd, bus_size_t offset, type value) \
1069 { \
1070 bus_space_write_##width(tag, hnd, offset, value); \
1071 return (0); \
1072 }
1073BUS_POKE_FUNC(1, uint8_t)
1074BUS_POKE_FUNC(2, uint16_t)
1075BUS_POKE_FUNC(4, uint32_t)
1076#ifdef __amd64__
1077BUS_POKE_FUNC(8, uint64_t)
1078#endif
1079
1080#endif /* !SAN_NEEDS_INTERCEPTORS && SAN_RUNTIME */
1081
1082#endif /* !_MACHINE_BUS_H_ */