From fe13ba28e3e37a5d53186444d9516d58046a66c7 Mon Sep 17 00:00:00 2001 From: zarazaex69 Date: Sat, 25 Apr 2026 21:22:13 +0300 Subject: [PATCH] feat(b): Add support for RGBA frame format in B visual codec --- go.mod | 2 +- go.sum | 4 +- internal/transport/videochannel/ffmpeg.go | 62 ++++++++++------ internal/transport/videochannel/transport.go | 4 +- internal/transport/videochannel/visual_b.go | 72 ++++++------------- .../transport/videochannel/visual_b_test.go | 47 ++++++++++++ 6 files changed, 115 insertions(+), 76 deletions(-) create mode 100644 internal/transport/videochannel/visual_b_test.go diff --git a/go.mod b/go.mod index cbd30ed..e019416 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/makiuchi-d/gozxing v0.1.1 github.com/pion/rtp v1.10.1 github.com/pion/webrtc/v4 v4.2.11 - github.com/zarazaex69/b v0.0.0-20260422171520-7eb386d13bda + github.com/zarazaex69/b v0.0.0-20260423064626-c0bd20863b89 golang.org/x/crypto v0.50.0 golang.org/x/mobile v0.0.0-20260410095206-2cfb76559b7b ) diff --git a/go.sum b/go.sum index 933b33f..22a34eb 100644 --- a/go.sum +++ b/go.sum @@ -197,8 +197,8 @@ github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1: github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -github.com/zarazaex69/b v0.0.0-20260422171520-7eb386d13bda h1:WVp2h2eFtWu/VU6HTq1Xh0VI/y5jI5svlxXIkIbXGEM= -github.com/zarazaex69/b v0.0.0-20260422171520-7eb386d13bda/go.mod h1:OUqzZNoXsg+ccaiAnSe0t4f8qc0W/cFx6io0lWsE1Gw= +github.com/zarazaex69/b v0.0.0-20260423064626-c0bd20863b89 h1:ytA0RfQZTYfjqFA9lBJMX1DTnXpTuKg0nf4udgdpunE= +github.com/zarazaex69/b v0.0.0-20260423064626-c0bd20863b89/go.mod h1:OUqzZNoXsg+ccaiAnSe0t4f8qc0W/cFx6io0lWsE1Gw= github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= diff --git a/internal/transport/videochannel/ffmpeg.go b/internal/transport/videochannel/ffmpeg.go index fc9a4ca..ff96a16 100644 --- a/internal/transport/videochannel/ffmpeg.go +++ b/internal/transport/videochannel/ffmpeg.go @@ -127,13 +127,14 @@ type ffmpegEncoder struct { frames chan []byte width int height int + frameSize int closed atomic.Bool closeOnce sync.Once errMu sync.Mutex err error } -func newFFmpegEncoder(spec codecSpec, width, height, fps int, bitrate, hw string) (*ffmpegEncoder, error) { +func newFFmpegEncoder(spec codecSpec, width, height, fps int, bitrate, hw, visualCodec string) (*ffmpegEncoder, error) { if _, err := exec.LookPath("ffmpeg"); err != nil { return nil, ErrFFmpegUnavailable } @@ -155,9 +156,16 @@ func newFFmpegEncoder(spec codecSpec, width, height, fps int, bitrate, hw string } } + inputPixFmt := "gray" + frameSize := width * height + if visualCodec == "b" { + inputPixFmt = "rgba" + frameSize = width * height * 4 + } + args = append(args, "-f", "rawvideo", - "-pix_fmt", "gray", + "-pix_fmt", inputPixFmt, "-video_size", fmt.Sprintf("%dx%d", width, height), "-framerate", fmt.Sprintf("%d", fps), "-i", "pipe:0", @@ -202,12 +210,13 @@ func newFFmpegEncoder(spec codecSpec, width, height, fps int, bitrate, hw string } enc := &ffmpegEncoder{ - cmd: cmd, - stdin: stdin, - stderr: stderr, - frames: make(chan []byte, 8), - width: width, - height: height, + cmd: cmd, + stdin: stdin, + stderr: stderr, + frames: make(chan []byte, 8), + width: width, + height: height, + frameSize: frameSize, } if spec.mimeType == webrtc.MimeTypeH264 { @@ -219,8 +228,8 @@ func newFFmpegEncoder(spec codecSpec, width, height, fps int, bitrate, hw string } func (e *ffmpegEncoder) EncodeFrame(frame []byte) ([]byte, error) { - if len(frame) != e.width*e.height { - return nil, fmt.Errorf("unexpected encoder frame size: %d", len(frame)) + if len(frame) != e.frameSize { + return nil, fmt.Errorf("unexpected encoder frame size: %d (expected %d)", len(frame), e.frameSize) } if err := e.processErr(); err != nil { return nil, err @@ -329,13 +338,14 @@ type ffmpegDecoder struct { frames chan []byte pts uint64 mimeType string + frameSize int closed atomic.Bool closeOnce sync.Once errMu sync.Mutex err error } -func newFFmpegDecoder(spec codecSpec, width, height, fps int, hw string) (*ffmpegDecoder, error) { +func newFFmpegDecoder(spec codecSpec, width, height, fps int, hw, visualCodec string) (*ffmpegDecoder, error) { if _, err := exec.LookPath("ffmpeg"); err != nil { return nil, ErrFFmpegUnavailable } @@ -352,6 +362,13 @@ func newFFmpegDecoder(spec codecSpec, width, height, fps int, hw string) (*ffmpe } } + outputPixFmt := "gray" + frameSize := width * height + if visualCodec == "b" { + outputPixFmt = "rgba" + frameSize = width * height * 4 + } + args := []string{"-loglevel", "info"} if spec.mimeType == webrtc.MimeTypeH264 { args = append(args, "-f", "h264") @@ -359,13 +376,14 @@ func newFFmpegDecoder(spec codecSpec, width, height, fps int, hw string) (*ffmpe args = append(args, "-f", "ivf") } + vfFilter := fmt.Sprintf("scale=%d:%d:flags=neighbor,format=%s", width, height, outputPixFmt) args = append(args, "-flags", "low_delay", "-vcodec", decoderName, "-i", "pipe:0", "-an", - "-vf", fmt.Sprintf("scale=%d:%d:flags=neighbor,format=gray", width, height), - "-pix_fmt", "gray", + "-vf", vfFilter, + "-pix_fmt", outputPixFmt, "-f", "rawvideo", "pipe:1", ) @@ -387,11 +405,12 @@ func newFFmpegDecoder(spec codecSpec, width, height, fps int, hw string) (*ffmpe } dec := &ffmpegDecoder{ - cmd: cmd, - stdin: stdin, - stderr: stderr, - frames: make(chan []byte, 32), - mimeType: spec.mimeType, + cmd: cmd, + stdin: stdin, + stderr: stderr, + frames: make(chan []byte, 32), + mimeType: spec.mimeType, + frameSize: frameSize, } if spec.mimeType != webrtc.MimeTypeH264 { @@ -401,7 +420,7 @@ func newFFmpegDecoder(spec codecSpec, width, height, fps int, hw string) (*ffmpe } } - go dec.readRawFrames(stdout, width, height) + go dec.readRawFrames(stdout) return dec, nil } @@ -446,10 +465,9 @@ func (d *ffmpegDecoder) Close() error { return nil } -func (d *ffmpegDecoder) readRawFrames(stdout io.Reader, width, height int) { +func (d *ffmpegDecoder) readRawFrames(stdout io.Reader) { defer close(d.frames) - logicalFrameBytes := width * height - buf := make([]byte, logicalFrameBytes) + buf := make([]byte, d.frameSize) for { if _, err := io.ReadFull(stdout, buf); err != nil { if !d.closed.Load() { diff --git a/internal/transport/videochannel/transport.go b/internal/transport/videochannel/transport.go index 32aa233..a90cff2 100644 --- a/internal/transport/videochannel/transport.go +++ b/internal/transport/videochannel/transport.go @@ -135,7 +135,7 @@ func (p *streamTransport) Connect(ctx context.Context) error { connectCtx, cancel := context.WithTimeout(ctx, defaultConnectTimeout) defer cancel() - encoder, err := newFFmpegEncoder(p.codec, p.videoW, p.videoH, p.videoFPS, p.videoBitrate, p.videoHW) + encoder, err := newFFmpegEncoder(p.codec, p.videoW, p.videoH, p.videoFPS, p.videoBitrate, p.videoHW, p.videoCodec) if err != nil { return err } @@ -349,7 +349,7 @@ func (p *streamTransport) handleRemoteTrack(track *webrtc.TrackRemote, _ *webrtc return } - decoder, err := newFFmpegDecoder(codec, p.videoW, p.videoH, p.videoFPS, p.videoHW) + decoder, err := newFFmpegDecoder(codec, p.videoW, p.videoH, p.videoFPS, p.videoHW, p.videoCodec) if err != nil { logger.Warnf("videochannel decoder init failed: %v", err) return diff --git a/internal/transport/videochannel/visual_b.go b/internal/transport/videochannel/visual_b.go index b8b9701..8605c5c 100644 --- a/internal/transport/videochannel/visual_b.go +++ b/internal/transport/videochannel/visual_b.go @@ -9,14 +9,16 @@ import ( ) func renderVisualFrameB(payload []byte, width, height int) ([]byte, error) { - logicalFrameBytes := width * height - frame := make([]byte, logicalFrameBytes) - for i := range frame { - frame[i] = 0xff + rgba := make([]byte, width*height*4) + for i := 0; i < len(rgba); i += 4 { + rgba[i] = 0xff + rgba[i+1] = 0xff + rgba[i+2] = 0xff + rgba[i+3] = 0xff } if len(payload) == 0 { - return frame, nil + return rgba, nil } cfg := b.DefaultConfig() @@ -27,63 +29,35 @@ func renderVisualFrameB(payload []byte, width, height int) ([]byte, error) { bmpW := int(result.Width) bmpH := int(result.Height) - - scaleW := width / bmpW - scaleH := height / bmpH - scale := scaleW - if scaleH < scale { - scale = scaleH - } - if scale < 1 { - scale = 1 - } - - totalW := bmpW * scale - totalH := bmpH * scale - offsetX := (width - totalW) / 2 - offsetY := (height - totalH) / 2 + offsetX := (width - bmpW) / 2 + offsetY := (height - bmpH) / 2 for y := 0; y < bmpH; y++ { for x := 0; x < bmpW; x++ { - idx := (y*bmpW + x) * 4 - r := result.RGBA[idx] - g := result.RGBA[idx+1] - bb := result.RGBA[idx+2] - - gray := uint8((int(r) + int(g) + int(bb)) / 3) - - for sy := 0; sy < scale; sy++ { - for sx := 0; sx < scale; sx++ { - pixelX := offsetX + (x * scale) + sx - pixelY := offsetY + (y * scale) + sy - if pixelX < width && pixelY < height { - frame[pixelY*width+pixelX] = gray - } - } + srcIdx := (y*bmpW + x) * 4 + pixelX := offsetX + x + pixelY := offsetY + y + if pixelX >= 0 && pixelX < width && pixelY >= 0 && pixelY < height { + dstIdx := (pixelY*width + pixelX) * 4 + rgba[dstIdx] = result.RGBA[srcIdx] + rgba[dstIdx+1] = result.RGBA[srcIdx+1] + rgba[dstIdx+2] = result.RGBA[srcIdx+2] + rgba[dstIdx+3] = result.RGBA[srcIdx+3] } } } - return frame, nil + return rgba, nil } func extractVisualPayloadB(frame []byte, width, height int) ([]byte, error) { - logicalFrameBytes := width * height - if len(frame) != logicalFrameBytes { - return nil, fmt.Errorf("unexpected frame size: %d (expected %dx%d=%d)", len(frame), width, height, logicalFrameBytes) - } - - rgba := make([]byte, width*height*4) - for i := 0; i < width*height; i++ { - gray := frame[i] - rgba[i*4] = gray - rgba[i*4+1] = gray - rgba[i*4+2] = gray - rgba[i*4+3] = 255 + expectedSize := width * height * 4 + if len(frame) != expectedSize { + return nil, fmt.Errorf("unexpected frame size: %d (expected %dx%dx4=%d)", len(frame), width, height, expectedSize) } cfg := b.DefaultConfig() - decoded, err := b.Decode(rgba, uint32(width), uint32(height), cfg) + decoded, err := b.Decode(frame, uint32(width), uint32(height), cfg) if err != nil { return nil, nil } diff --git a/internal/transport/videochannel/visual_b_test.go b/internal/transport/videochannel/visual_b_test.go new file mode 100644 index 0000000..564b3df --- /dev/null +++ b/internal/transport/videochannel/visual_b_test.go @@ -0,0 +1,47 @@ +//go:build b + +package videochannel + +import ( + "bytes" + "testing" +) + +func TestBCodecRoundtrip(t *testing.T) { + data := []byte("Hello JABCode test 123456789012345678901234567890") + width, height := 480, 480 + + frame, err := renderVisualFrameB(data, width, height) + if err != nil { + t.Fatalf("renderVisualFrameB failed: %v", err) + } + expectedSize := width * height * 4 + if len(frame) != expectedSize { + t.Fatalf("unexpected frame size: %d, want %d", len(frame), expectedSize) + } + + payload, err := extractVisualPayloadB(frame, width, height) + if err != nil { + t.Fatalf("extractVisualPayloadB failed: %v", err) + } + if payload == nil { + t.Fatal("extractVisualPayloadB returned nil payload") + } + + if !bytes.Equal(payload, data) { + t.Fatalf("roundtrip mismatch:\noriginal: %q\ndecoded: %q", string(data), string(payload)) + } +} + +func TestBCodecEmptyPayload(t *testing.T) { + width, height := 480, 480 + + frame, err := renderVisualFrameB(nil, width, height) + if err != nil { + t.Fatalf("renderVisualFrameB with empty payload failed: %v", err) + } + expectedSize := width * height * 4 + if len(frame) != expectedSize { + t.Fatalf("unexpected frame size: %d", len(frame)) + } +}