GitHub/olcrtc
1
0
Fork 0
mirror of https://github.com/openlibrecommunity/olcrtc.git synced 2026-05-26 07:08:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(b): Add support for RGBA frame format in B visual codec

Browse Source
This commit is contained in:
zarazaex69
2026-04-25 21:22:13 +03:00
parent 5c36e0d95a
commit fe13ba28e3
6 changed files with 115 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
go.mod
View File

@@ -11,7 +11,7 @@ require (
github.com/makiuchi-d/gozxing v0.1.1
github.com/pion/rtp v1.10.1
github.com/pion/webrtc/v4 v4.2.11
github.com/zarazaex69/b v0.0.0-20260422171520-7eb386d13bda
github.com/zarazaex69/b v0.0.0-20260423064626-c0bd20863b89
golang.org/x/crypto v0.50.0
golang.org/x/mobile v0.0.0-20260410095206-2cfb76559b7b
)

4
go.sum
View File

@@ -197,8 +197,8 @@ github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:
github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74=
github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
github.com/zarazaex69/b v0.0.0-20260422171520-7eb386d13bda h1:WVp2h2eFtWu/VU6HTq1Xh0VI/y5jI5svlxXIkIbXGEM=
github.com/zarazaex69/b v0.0.0-20260422171520-7eb386d13bda/go.mod h1:OUqzZNoXsg+ccaiAnSe0t4f8qc0W/cFx6io0lWsE1Gw=
github.com/zarazaex69/b v0.0.0-20260423064626-c0bd20863b89 h1:ytA0RfQZTYfjqFA9lBJMX1DTnXpTuKg0nf4udgdpunE=
github.com/zarazaex69/b v0.0.0-20260423064626-c0bd20863b89/go.mod h1:OUqzZNoXsg+ccaiAnSe0t4f8qc0W/cFx6io0lWsE1Gw=
github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs=

62
internal/transport/videochannel/ffmpeg.go
View File

@@ -127,13 +127,14 @@ type ffmpegEncoder struct {
frames chan []byte
width int
height int
frameSize int
closed atomic.Bool
closeOnce sync.Once
errMu sync.Mutex
err error
}
func newFFmpegEncoder(spec codecSpec, width, height, fps int, bitrate, hw string) (*ffmpegEncoder, error) {
func newFFmpegEncoder(spec codecSpec, width, height, fps int, bitrate, hw, visualCodec string) (*ffmpegEncoder, error) {
if _, err := exec.LookPath("ffmpeg"); err != nil {
return nil, ErrFFmpegUnavailable
}
@@ -155,9 +156,16 @@ func newFFmpegEncoder(spec codecSpec, width, height, fps int, bitrate, hw string
}
}
inputPixFmt := "gray"
frameSize := width * height
if visualCodec == "b" {
inputPixFmt = "rgba"
frameSize = width * height * 4
}
args = append(args,
"-f", "rawvideo",
"-pix_fmt", "gray",
"-pix_fmt", inputPixFmt,
"-video_size", fmt.Sprintf("%dx%d", width, height),
"-framerate", fmt.Sprintf("%d", fps),
"-i", "pipe:0",
@@ -202,12 +210,13 @@ func newFFmpegEncoder(spec codecSpec, width, height, fps int, bitrate, hw string
}
enc := &ffmpegEncoder{
cmd: cmd,
stdin: stdin,
stderr: stderr,
frames: make(chan []byte, 8),
width: width,
height: height,
cmd: cmd,
stdin: stdin,
stderr: stderr,
frames: make(chan []byte, 8),
width: width,
height: height,
frameSize: frameSize,
}
if spec.mimeType == webrtc.MimeTypeH264 {
@@ -219,8 +228,8 @@ func newFFmpegEncoder(spec codecSpec, width, height, fps int, bitrate, hw string
}
func (e *ffmpegEncoder) EncodeFrame(frame []byte) ([]byte, error) {
if len(frame) != e.width*e.height {
return nil, fmt.Errorf("unexpected encoder frame size: %d", len(frame))
if len(frame) != e.frameSize {
return nil, fmt.Errorf("unexpected encoder frame size: %d (expected %d)", len(frame), e.frameSize)
}
if err := e.processErr(); err != nil {
return nil, err
@@ -329,13 +338,14 @@ type ffmpegDecoder struct {
frames chan []byte
pts uint64
mimeType string
frameSize int
closed atomic.Bool
closeOnce sync.Once
errMu sync.Mutex
err error
}
func newFFmpegDecoder(spec codecSpec, width, height, fps int, hw string) (*ffmpegDecoder, error) {
func newFFmpegDecoder(spec codecSpec, width, height, fps int, hw, visualCodec string) (*ffmpegDecoder, error) {
if _, err := exec.LookPath("ffmpeg"); err != nil {
return nil, ErrFFmpegUnavailable
}
@@ -352,6 +362,13 @@ func newFFmpegDecoder(spec codecSpec, width, height, fps int, hw string) (*ffmpe
}
}
outputPixFmt := "gray"
frameSize := width * height
if visualCodec == "b" {
outputPixFmt = "rgba"
frameSize = width * height * 4
}
args := []string{"-loglevel", "info"}
if spec.mimeType == webrtc.MimeTypeH264 {
args = append(args, "-f", "h264")
@@ -359,13 +376,14 @@ func newFFmpegDecoder(spec codecSpec, width, height, fps int, hw string) (*ffmpe
args = append(args, "-f", "ivf")
}
vfFilter := fmt.Sprintf("scale=%d:%d:flags=neighbor,format=%s", width, height, outputPixFmt)
args = append(args,
"-flags", "low_delay",
"-vcodec", decoderName,
"-i", "pipe:0",
"-an",
"-vf", fmt.Sprintf("scale=%d:%d:flags=neighbor,format=gray", width, height),
"-pix_fmt", "gray",
"-vf", vfFilter,
"-pix_fmt", outputPixFmt,
"-f", "rawvideo",
"pipe:1",
)
@@ -387,11 +405,12 @@ func newFFmpegDecoder(spec codecSpec, width, height, fps int, hw string) (*ffmpe
}
dec := &ffmpegDecoder{
cmd: cmd,
stdin: stdin,
stderr: stderr,
frames: make(chan []byte, 32),
mimeType: spec.mimeType,
cmd: cmd,
stdin: stdin,
stderr: stderr,
frames: make(chan []byte, 32),
mimeType: spec.mimeType,
frameSize: frameSize,
}
if spec.mimeType != webrtc.MimeTypeH264 {
@@ -401,7 +420,7 @@ func newFFmpegDecoder(spec codecSpec, width, height, fps int, hw string) (*ffmpe
}
}
go dec.readRawFrames(stdout, width, height)
go dec.readRawFrames(stdout)
return dec, nil
}
@@ -446,10 +465,9 @@ func (d *ffmpegDecoder) Close() error {
return nil
}
func (d *ffmpegDecoder) readRawFrames(stdout io.Reader, width, height int) {
func (d *ffmpegDecoder) readRawFrames(stdout io.Reader) {
defer close(d.frames)
logicalFrameBytes := width * height
buf := make([]byte, logicalFrameBytes)
buf := make([]byte, d.frameSize)
for {
if _, err := io.ReadFull(stdout, buf); err != nil {
if !d.closed.Load() {

4
internal/transport/videochannel/transport.go
View File

@@ -135,7 +135,7 @@ func (p *streamTransport) Connect(ctx context.Context) error {
connectCtx, cancel := context.WithTimeout(ctx, defaultConnectTimeout)
defer cancel()
encoder, err := newFFmpegEncoder(p.codec, p.videoW, p.videoH, p.videoFPS, p.videoBitrate, p.videoHW)
encoder, err := newFFmpegEncoder(p.codec, p.videoW, p.videoH, p.videoFPS, p.videoBitrate, p.videoHW, p.videoCodec)
if err != nil {
return err
}
@@ -349,7 +349,7 @@ func (p *streamTransport) handleRemoteTrack(track *webrtc.TrackRemote, _ *webrtc
return
}
decoder, err := newFFmpegDecoder(codec, p.videoW, p.videoH, p.videoFPS, p.videoHW)
decoder, err := newFFmpegDecoder(codec, p.videoW, p.videoH, p.videoFPS, p.videoHW, p.videoCodec)
if err != nil {
logger.Warnf("videochannel decoder init failed: %v", err)
return

72
internal/transport/videochannel/visual_b.go
View File

@@ -9,14 +9,16 @@ import (
)
func renderVisualFrameB(payload []byte, width, height int) ([]byte, error) {
logicalFrameBytes := width * height
frame := make([]byte, logicalFrameBytes)
for i := range frame {
frame[i] = 0xff
rgba := make([]byte, width*height*4)
for i := 0; i < len(rgba); i += 4 {
rgba[i] = 0xff
rgba[i+1] = 0xff
rgba[i+2] = 0xff
rgba[i+3] = 0xff
}
if len(payload) == 0 {
return frame, nil
return rgba, nil
}
cfg := b.DefaultConfig()
@@ -27,63 +29,35 @@ func renderVisualFrameB(payload []byte, width, height int) ([]byte, error) {
bmpW := int(result.Width)
bmpH := int(result.Height)
scaleW := width / bmpW
scaleH := height / bmpH
scale := scaleW
if scaleH < scale {
scale = scaleH
}
if scale < 1 {
scale = 1
}
totalW := bmpW * scale
totalH := bmpH * scale
offsetX := (width - totalW) / 2
offsetY := (height - totalH) / 2
offsetX := (width - bmpW) / 2
offsetY := (height - bmpH) / 2
for y := 0; y < bmpH; y++ {
for x := 0; x < bmpW; x++ {
idx := (y*bmpW + x) * 4
r := result.RGBA[idx]
g := result.RGBA[idx+1]
bb := result.RGBA[idx+2]
gray := uint8((int(r) + int(g) + int(bb)) / 3)
for sy := 0; sy < scale; sy++ {
for sx := 0; sx < scale; sx++ {
pixelX := offsetX + (x * scale) + sx
pixelY := offsetY + (y * scale) + sy
if pixelX < width && pixelY < height {
frame[pixelY*width+pixelX] = gray
}
}
srcIdx := (y*bmpW + x) * 4
pixelX := offsetX + x
pixelY := offsetY + y
if pixelX >= 0 && pixelX < width && pixelY >= 0 && pixelY < height {
dstIdx := (pixelY*width + pixelX) * 4
rgba[dstIdx] = result.RGBA[srcIdx]
rgba[dstIdx+1] = result.RGBA[srcIdx+1]
rgba[dstIdx+2] = result.RGBA[srcIdx+2]
rgba[dstIdx+3] = result.RGBA[srcIdx+3]
}
}
}
return frame, nil
return rgba, nil
}
func extractVisualPayloadB(frame []byte, width, height int) ([]byte, error) {
logicalFrameBytes := width * height
if len(frame) != logicalFrameBytes {
return nil, fmt.Errorf("unexpected frame size: %d (expected %dx%d=%d)", len(frame), width, height, logicalFrameBytes)
}
rgba := make([]byte, width*height*4)
for i := 0; i < width*height; i++ {
gray := frame[i]
rgba[i*4] = gray
rgba[i*4+1] = gray
rgba[i*4+2] = gray
rgba[i*4+3] = 255
expectedSize := width * height * 4
if len(frame) != expectedSize {
return nil, fmt.Errorf("unexpected frame size: %d (expected %dx%dx4=%d)", len(frame), width, height, expectedSize)
}
cfg := b.DefaultConfig()
decoded, err := b.Decode(rgba, uint32(width), uint32(height), cfg)
decoded, err := b.Decode(frame, uint32(width), uint32(height), cfg)
if err != nil {
return nil, nil
}

47
internal/transport/videochannel/visual_b_test.go Normal file
View File

@@ -0,0 +1,47 @@
//go:build b
package videochannel
import (
"bytes"
"testing"
)
func TestBCodecRoundtrip(t *testing.T) {
data := []byte("Hello JABCode test 123456789012345678901234567890")
width, height := 480, 480
frame, err := renderVisualFrameB(data, width, height)
if err != nil {
t.Fatalf("renderVisualFrameB failed: %v", err)
}
expectedSize := width * height * 4
if len(frame) != expectedSize {
t.Fatalf("unexpected frame size: %d, want %d", len(frame), expectedSize)
}
payload, err := extractVisualPayloadB(frame, width, height)
if err != nil {
t.Fatalf("extractVisualPayloadB failed: %v", err)
}
if payload == nil {
t.Fatal("extractVisualPayloadB returned nil payload")
}
if !bytes.Equal(payload, data) {
t.Fatalf("roundtrip mismatch:\noriginal: %q\ndecoded: %q", string(data), string(payload))
}
}
func TestBCodecEmptyPayload(t *testing.T) {
width, height := 480, 480
frame, err := renderVisualFrameB(nil, width, height)
if err != nil {
t.Fatalf("renderVisualFrameB with empty payload failed: %v", err)
}
expectedSize := width * height * 4
if len(frame) != expectedSize {
t.Fatalf("unexpected frame size: %d", len(frame))
}
}