feat(b): Add support for RGBA frame format in B visual codec

go.mod

@@ -11,7 +11,7 @@ require (
 	github.com/makiuchi-d/gozxing v0.1.1
 	github.com/pion/rtp v1.10.1
 	github.com/pion/webrtc/v4 v4.2.11
-	github.com/zarazaex69/b v0.0.0-20260422171520-7eb386d13bda
+	github.com/zarazaex69/b v0.0.0-20260423064626-c0bd20863b89
 	golang.org/x/crypto v0.50.0
 	golang.org/x/mobile v0.0.0-20260410095206-2cfb76559b7b
 )

go.sum

@@ -197,8 +197,8 @@ github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:
 github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74=
 github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
-github.com/zarazaex69/b v0.0.0-20260422171520-7eb386d13bda h1:WVp2h2eFtWu/VU6HTq1Xh0VI/y5jI5svlxXIkIbXGEM=
-github.com/zarazaex69/b v0.0.0-20260422171520-7eb386d13bda/go.mod h1:OUqzZNoXsg+ccaiAnSe0t4f8qc0W/cFx6io0lWsE1Gw=
+github.com/zarazaex69/b v0.0.0-20260423064626-c0bd20863b89 h1:ytA0RfQZTYfjqFA9lBJMX1DTnXpTuKg0nf4udgdpunE=
+github.com/zarazaex69/b v0.0.0-20260423064626-c0bd20863b89/go.mod h1:OUqzZNoXsg+ccaiAnSe0t4f8qc0W/cFx6io0lWsE1Gw=
 github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
 github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
 github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs=

internal/transport/videochannel/ffmpeg.go

@@ -127,13 +127,14 @@ type ffmpegEncoder struct {
 	frames    chan []byte
 	width     int
 	height    int
+	frameSize int
 	closed    atomic.Bool
 	closeOnce sync.Once
 	errMu     sync.Mutex
 	err       error
 }
 
-func newFFmpegEncoder(spec codecSpec, width, height, fps int, bitrate, hw string) (*ffmpegEncoder, error) {
+func newFFmpegEncoder(spec codecSpec, width, height, fps int, bitrate, hw, visualCodec string) (*ffmpegEncoder, error) {
 	if _, err := exec.LookPath("ffmpeg"); err != nil {
 		return nil, ErrFFmpegUnavailable
 	}
@@ -155,9 +156,16 @@ func newFFmpegEncoder(spec codecSpec, width, height, fps int, bitrate, hw string
 		}
 	}
 
+	inputPixFmt := "gray"
+	frameSize := width * height
+	if visualCodec == "b" {
+		inputPixFmt = "rgba"
+		frameSize = width * height * 4
+	}
+
 	args = append(args,
 		"-f", "rawvideo",
-		"-pix_fmt", "gray",
+		"-pix_fmt", inputPixFmt,
 		"-video_size", fmt.Sprintf("%dx%d", width, height),
 		"-framerate", fmt.Sprintf("%d", fps),
 		"-i", "pipe:0",
@@ -208,6 +216,7 @@ func newFFmpegEncoder(spec codecSpec, width, height, fps int, bitrate, hw string
 		frames:    make(chan []byte, 8),
 		width:     width,
 		height:    height,
+		frameSize: frameSize,
 	}
 
 	if spec.mimeType == webrtc.MimeTypeH264 {
@@ -219,8 +228,8 @@ func newFFmpegEncoder(spec codecSpec, width, height, fps int, bitrate, hw string
 }
 
 func (e *ffmpegEncoder) EncodeFrame(frame []byte) ([]byte, error) {
-	if len(frame) != e.width*e.height {
-		return nil, fmt.Errorf("unexpected encoder frame size: %d", len(frame))
+	if len(frame) != e.frameSize {
+		return nil, fmt.Errorf("unexpected encoder frame size: %d (expected %d)", len(frame), e.frameSize)
 	}
 	if err := e.processErr(); err != nil {
 		return nil, err
@@ -329,13 +338,14 @@ type ffmpegDecoder struct {
 	frames    chan []byte
 	pts       uint64
 	mimeType  string
+	frameSize int
 	closed    atomic.Bool
 	closeOnce sync.Once
 	errMu     sync.Mutex
 	err       error
 }
 
-func newFFmpegDecoder(spec codecSpec, width, height, fps int, hw string) (*ffmpegDecoder, error) {
+func newFFmpegDecoder(spec codecSpec, width, height, fps int, hw, visualCodec string) (*ffmpegDecoder, error) {
 	if _, err := exec.LookPath("ffmpeg"); err != nil {
 		return nil, ErrFFmpegUnavailable
 	}
@@ -352,6 +362,13 @@ func newFFmpegDecoder(spec codecSpec, width, height, fps int, hw string) (*ffmpe
 		}
 	}
 
+	outputPixFmt := "gray"
+	frameSize := width * height
+	if visualCodec == "b" {
+		outputPixFmt = "rgba"
+		frameSize = width * height * 4
+	}
+
 	args := []string{"-loglevel", "info"}
 	if spec.mimeType == webrtc.MimeTypeH264 {
 		args = append(args, "-f", "h264")
@@ -359,13 +376,14 @@ func newFFmpegDecoder(spec codecSpec, width, height, fps int, hw string) (*ffmpe
 		args = append(args, "-f", "ivf")
 	}
 
+	vfFilter := fmt.Sprintf("scale=%d:%d:flags=neighbor,format=%s", width, height, outputPixFmt)
 	args = append(args,
 		"-flags", "low_delay",
 		"-vcodec", decoderName,
 		"-i", "pipe:0",
 		"-an",
-		"-vf", fmt.Sprintf("scale=%d:%d:flags=neighbor,format=gray", width, height),
-		"-pix_fmt", "gray",
+		"-vf", vfFilter,
+		"-pix_fmt", outputPixFmt,
 		"-f", "rawvideo",
 		"pipe:1",
 	)
@@ -392,6 +410,7 @@ func newFFmpegDecoder(spec codecSpec, width, height, fps int, hw string) (*ffmpe
 		stderr:    stderr,
 		frames:    make(chan []byte, 32),
 		mimeType:  spec.mimeType,
+		frameSize: frameSize,
 	}
 
 	if spec.mimeType != webrtc.MimeTypeH264 {
@@ -401,7 +420,7 @@ func newFFmpegDecoder(spec codecSpec, width, height, fps int, hw string) (*ffmpe
 		}
 	}
 
-	go dec.readRawFrames(stdout, width, height)
+	go dec.readRawFrames(stdout)
 	return dec, nil
 }
 
@@ -446,10 +465,9 @@ func (d *ffmpegDecoder) Close() error {
 	return nil
 }
 
-func (d *ffmpegDecoder) readRawFrames(stdout io.Reader, width, height int) {
+func (d *ffmpegDecoder) readRawFrames(stdout io.Reader) {
 	defer close(d.frames)
-	logicalFrameBytes := width * height
-	buf := make([]byte, logicalFrameBytes)
+	buf := make([]byte, d.frameSize)
 	for {
 		if _, err := io.ReadFull(stdout, buf); err != nil {
 			if !d.closed.Load() {

internal/transport/videochannel/transport.go

@@ -135,7 +135,7 @@ func (p *streamTransport) Connect(ctx context.Context) error {
 	connectCtx, cancel := context.WithTimeout(ctx, defaultConnectTimeout)
 	defer cancel()
 
-	encoder, err := newFFmpegEncoder(p.codec, p.videoW, p.videoH, p.videoFPS, p.videoBitrate, p.videoHW)
+	encoder, err := newFFmpegEncoder(p.codec, p.videoW, p.videoH, p.videoFPS, p.videoBitrate, p.videoHW, p.videoCodec)
 	if err != nil {
 		return err
 	}
@@ -349,7 +349,7 @@ func (p *streamTransport) handleRemoteTrack(track *webrtc.TrackRemote, _ *webrtc
 		return
 	}
 
-	decoder, err := newFFmpegDecoder(codec, p.videoW, p.videoH, p.videoFPS, p.videoHW)
+	decoder, err := newFFmpegDecoder(codec, p.videoW, p.videoH, p.videoFPS, p.videoHW, p.videoCodec)
 	if err != nil {
 		logger.Warnf("videochannel decoder init failed: %v", err)
 		return

internal/transport/videochannel/visual_b.go

@@ -9,14 +9,16 @@ import (
 )
 
 func renderVisualFrameB(payload []byte, width, height int) ([]byte, error) {
-	logicalFrameBytes := width * height
-	frame := make([]byte, logicalFrameBytes)
-	for i := range frame {
-		frame[i] = 0xff
+	rgba := make([]byte, width*height*4)
+	for i := 0; i < len(rgba); i += 4 {
+		rgba[i] = 0xff
+		rgba[i+1] = 0xff
+		rgba[i+2] = 0xff
+		rgba[i+3] = 0xff
 	}
 
 	if len(payload) == 0 {
-		return frame, nil
+		return rgba, nil
 	}
 
 	cfg := b.DefaultConfig()
@@ -27,63 +29,35 @@ func renderVisualFrameB(payload []byte, width, height int) ([]byte, error) {
 
 	bmpW := int(result.Width)
 	bmpH := int(result.Height)
-
-	scaleW := width / bmpW
-	scaleH := height / bmpH
-	scale := scaleW
-	if scaleH < scale {
-		scale = scaleH
-	}
-	if scale < 1 {
-		scale = 1
-	}
-
-	totalW := bmpW * scale
-	totalH := bmpH * scale
-	offsetX := (width - totalW) / 2
-	offsetY := (height - totalH) / 2
+	offsetX := (width - bmpW) / 2
+	offsetY := (height - bmpH) / 2
 
 	for y := 0; y < bmpH; y++ {
 		for x := 0; x < bmpW; x++ {
-			idx := (y*bmpW + x) * 4
-			r := result.RGBA[idx]
-			g := result.RGBA[idx+1]
-			bb := result.RGBA[idx+2]
-
-			gray := uint8((int(r) + int(g) + int(bb)) / 3)
-
-			for sy := 0; sy < scale; sy++ {
-				for sx := 0; sx < scale; sx++ {
-					pixelX := offsetX + (x * scale) + sx
-					pixelY := offsetY + (y * scale) + sy
-					if pixelX < width && pixelY < height {
-						frame[pixelY*width+pixelX] = gray
-					}
-				}
+			srcIdx := (y*bmpW + x) * 4
+			pixelX := offsetX + x
+			pixelY := offsetY + y
+			if pixelX >= 0 && pixelX < width && pixelY >= 0 && pixelY < height {
+				dstIdx := (pixelY*width + pixelX) * 4
+				rgba[dstIdx] = result.RGBA[srcIdx]
+				rgba[dstIdx+1] = result.RGBA[srcIdx+1]
+				rgba[dstIdx+2] = result.RGBA[srcIdx+2]
+				rgba[dstIdx+3] = result.RGBA[srcIdx+3]
 			}
 		}
 	}
 
-	return frame, nil
+	return rgba, nil
 }
 
 func extractVisualPayloadB(frame []byte, width, height int) ([]byte, error) {
-	logicalFrameBytes := width * height
-	if len(frame) != logicalFrameBytes {
-		return nil, fmt.Errorf("unexpected frame size: %d (expected %dx%d=%d)", len(frame), width, height, logicalFrameBytes)
-	}
-
-	rgba := make([]byte, width*height*4)
-	for i := 0; i < width*height; i++ {
-		gray := frame[i]
-		rgba[i*4] = gray
-		rgba[i*4+1] = gray
-		rgba[i*4+2] = gray
-		rgba[i*4+3] = 255
+	expectedSize := width * height * 4
+	if len(frame) != expectedSize {
+		return nil, fmt.Errorf("unexpected frame size: %d (expected %dx%dx4=%d)", len(frame), width, height, expectedSize)
 	}
 
 	cfg := b.DefaultConfig()
-	decoded, err := b.Decode(rgba, uint32(width), uint32(height), cfg)
+	decoded, err := b.Decode(frame, uint32(width), uint32(height), cfg)
 	if err != nil {
 		return nil, nil
 	}

internal/transport/videochannel/visual_b_test.go

@@ -0,0 +1,47 @@
+//go:build b
+
+package videochannel
+
+import (
+	"bytes"
+	"testing"
+)
+
+func TestBCodecRoundtrip(t *testing.T) {
+	data := []byte("Hello JABCode test 123456789012345678901234567890")
+	width, height := 480, 480
+
+	frame, err := renderVisualFrameB(data, width, height)
+	if err != nil {
+		t.Fatalf("renderVisualFrameB failed: %v", err)
+	}
+	expectedSize := width * height * 4
+	if len(frame) != expectedSize {
+		t.Fatalf("unexpected frame size: %d, want %d", len(frame), expectedSize)
+	}
+
+	payload, err := extractVisualPayloadB(frame, width, height)
+	if err != nil {
+		t.Fatalf("extractVisualPayloadB failed: %v", err)
+	}
+	if payload == nil {
+		t.Fatal("extractVisualPayloadB returned nil payload")
+	}
+
+	if !bytes.Equal(payload, data) {
+		t.Fatalf("roundtrip mismatch:\noriginal: %q\ndecoded:  %q", string(data), string(payload))
+	}
+}
+
+func TestBCodecEmptyPayload(t *testing.T) {
+	width, height := 480, 480
+
+	frame, err := renderVisualFrameB(nil, width, height)
+	if err != nil {
+		t.Fatalf("renderVisualFrameB with empty payload failed: %v", err)
+	}
+	expectedSize := width * height * 4
+	if len(frame) != expectedSize {
+		t.Fatalf("unexpected frame size: %d", len(frame))
+	}
+}