zero_copy_linux.go

     1  // Copyright 2020 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package os
     6  
     7  import (
     8  	"internal/poll"
     9  	"io"
    10  	"syscall"
    11  )
    12  
    13  var (
    14  	pollCopyFileRange = poll.CopyFileRange
    15  	pollSplice        = poll.Splice
    16  )
    17  
    18  func (f *File) writeTo(w io.Writer) (written int64, handled bool, err error) {
    19  	pfd, network := getPollFDAndNetwork(w)
    20  	// TODO(panjf2000): same as File.spliceToFile.
    21  	if pfd == nil || !pfd.IsStream || !isUnixOrTCP(string(network)) {
    22  		return
    23  	}
    24  
    25  	sc, err := f.SyscallConn()
    26  	if err != nil {
    27  		return
    28  	}
    29  
    30  	rerr := sc.Read(func(fd uintptr) (done bool) {
    31  		written, err, handled = poll.SendFile(pfd, int(fd), 0)
    32  		return true
    33  	})
    34  
    35  	if err == nil {
    36  		err = rerr
    37  	}
    38  
    39  	return written, handled, wrapSyscallError("sendfile", err)
    40  }
    41  
    42  func (f *File) readFrom(r io.Reader) (written int64, handled bool, err error) {
    43  	// Neither copy_file_range(2) nor splice(2) supports destinations opened with
    44  	// O_APPEND, so don't bother to try zero-copy with these system calls.
    45  	//
    46  	// Visit https://man7.org/linux/man-pages/man2/copy_file_range.2.html#ERRORS and
    47  	// https://man7.org/linux/man-pages/man2/splice.2.html#ERRORS for details.
    48  	if f.appendMode {
    49  		return 0, false, nil
    50  	}
    51  
    52  	written, handled, err = f.copyFileRange(r)
    53  	if handled {
    54  		return
    55  	}
    56  	return f.spliceToFile(r)
    57  }
    58  
    59  func (f *File) spliceToFile(r io.Reader) (written int64, handled bool, err error) {
    60  	var (
    61  		remain int64
    62  		lr     *io.LimitedReader
    63  	)
    64  	if lr, r, remain = tryLimitedReader(r); remain <= 0 {
    65  		return 0, true, nil
    66  	}
    67  
    68  	pfd, _ := getPollFDAndNetwork(r)
    69  	// TODO(panjf2000): run some tests to see if we should unlock the non-streams for splice.
    70  	// Streams benefit the most from the splice(2), non-streams are not even supported in old kernels
    71  	// where splice(2) will just return EINVAL; newer kernels support non-streams like UDP, but I really
    72  	// doubt that splice(2) could help non-streams, cuz they usually send small frames respectively
    73  	// and one splice call would result in one frame.
    74  	// splice(2) is suitable for large data but the generation of fragments defeats its edge here.
    75  	// Therefore, don't bother to try splice if the r is not a streaming descriptor.
    76  	if pfd == nil || !pfd.IsStream {
    77  		return
    78  	}
    79  
    80  	written, handled, err = pollSplice(&f.pfd, pfd, remain)
    81  
    82  	if lr != nil {
    83  		lr.N = remain - written
    84  	}
    85  
    86  	return written, handled, wrapSyscallError("splice", err)
    87  }
    88  
    89  func (f *File) copyFileRange(r io.Reader) (written int64, handled bool, err error) {
    90  	var (
    91  		remain int64
    92  		lr     *io.LimitedReader
    93  	)
    94  	if lr, r, remain = tryLimitedReader(r); remain <= 0 {
    95  		return 0, true, nil
    96  	}
    97  
    98  	var src *File
    99  	switch v := r.(type) {
   100  	case *File:
   101  		src = v
   102  	case fileWithoutWriteTo:
   103  		src = v.File
   104  	default:
   105  		return 0, false, nil
   106  	}
   107  
   108  	if src.checkValid("ReadFrom") != nil {
   109  		// Avoid returning the error as we report handled as false,
   110  		// leave further error handling as the responsibility of the caller.
   111  		return 0, false, nil
   112  	}
   113  
   114  	written, handled, err = pollCopyFileRange(&f.pfd, &src.pfd, remain)
   115  	if lr != nil {
   116  		lr.N -= written
   117  	}
   118  	return written, handled, wrapSyscallError("copy_file_range", err)
   119  }
   120  
   121  // getPollFDAndNetwork tries to get the poll.FD and network type from the given interface
   122  // by expecting the underlying type of i to be the implementation of syscall.Conn
   123  // that contains a *net.rawConn.
   124  func getPollFDAndNetwork(i any) (*poll.FD, poll.String) {
   125  	sc, ok := i.(syscall.Conn)
   126  	if !ok {
   127  		return nil, ""
   128  	}
   129  	rc, err := sc.SyscallConn()
   130  	if err != nil {
   131  		return nil, ""
   132  	}
   133  	irc, ok := rc.(interface {
   134  		PollFD() *poll.FD
   135  		Network() poll.String
   136  	})
   137  	if !ok {
   138  		return nil, ""
   139  	}
   140  	return irc.PollFD(), irc.Network()
   141  }
   142  
   143  func isUnixOrTCP(network string) bool {
   144  	switch network {
   145  	case "tcp", "tcp4", "tcp6", "unix":
   146  		return true
   147  	default:
   148  		return false
   149  	}
   150  }
   151
View as plain text