Posted on Jan 4, 2021 • Edited on Dec 17, 2021

Playing video in a Golang game

Please proceed here for a new version of the article. The method described below doesn't work anymore (and it's not really good either).

In videogames, we often need to show the player beautiful cinematics, either for ending or in the middle of the playthrough. To do it, we need to somehow load video frames and render them on the screen. Here's how to do that using Pixel game library and goav, Golang bindings for FFmpeg.

Initial setup

First we need to create a GLFW window for rendering OpenGL stuff:

package main import ( "fmt" "github.com/faiface/pixel" "github.com/faiface/pixel/pixelgl" colors "golang.org/x/image/colornames" ) const ( // WindowWidth is the width of the window. WindowWidth = 1280 // WindowHeight is the height of the window. WindowHeight = 720 ) func run() { // Create a new window. cfg := pixelgl.WindowConfig{ Title: "Pixel Rocks!", Bounds: pixel.R(0, 0, float64(WindowWidth), float64(WindowHeight)), VSync: false, } win, err := pixelgl.NewWindow(cfg) handleError(err) fps := 0 perSecond := time.Tick(time.Second) for !win.Closed() { win.Clear(colors.White) win.Update() // Show FPS in the window title. fps++ select { case <-perSecond: win.SetTitle(fmt.Sprintf("%s | FPS: %d", cfg.Title, fps)) fps = 0 default: } } } func main() { pixelgl.Run(run) } func handleError(err error) { if err != nil { panic(err) } }

Obtaining video frames

Now we need to obtain frames from the video stream of the file. goav provides an example code on how to do that. To make it work with Pixel game library, we need to set the frame decoding format to avcodec.AV_PIX_FMT_RGBA which is used by Pixel.

We also need a way to send the decoded frames to the renderer. For this task we will use a thread-safe frame buffer channel. First we should specify its size:

const ( FrameBufferSize = 1024 )

The greater the buffer size, the faster the frame transfer from the decoder to the renderer. Now to the channel creation:

frameBuffer := make(chan *pixel.PictureData, FrameBufferSize)

When the frame transfer is complete, we need to close the channel, but we also have to make sure the renderer got all the frames from the buffer. So here's the code for closing the frame buffer:

go func() { for { if len(frameBuffer) <= 0 { close(frameBuffer) break } } }()

The complete code for reading video frames is presented below:

func readVideoFrames(videoPath string) <-chan *pixel.PictureData { // Create a frame buffer. frameBuffer := make(chan *pixel.PictureData, FrameBufferSize) go func() { // Open a video file. pFormatContext := avformat.AvformatAllocContext() if avformat.AvformatOpenInput(&pFormatContext, videoPath, nil, nil) != 0 { fmt.Printf("Unable to open file %s\n", videoPath) os.Exit(1) } // Retrieve the stream information. if pFormatContext.AvformatFindStreamInfo(nil) < 0 { fmt.Println("Couldn't find stream information") os.Exit(1) } // Dump information about the video to stderr. pFormatContext.AvDumpFormat(0, videoPath, 0) // Find the first video stream for i := 0; i < int(pFormatContext.NbStreams()); i++ { switch pFormatContext.Streams()[i]. CodecParameters().AvCodecGetType() { case avformat.AVMEDIA_TYPE_VIDEO: // Get a pointer to the codec context for the video stream pCodecCtxOrig := pFormatContext.Streams()[i].Codec() // Find the decoder for the video stream pCodec := avcodec.AvcodecFindDecoder(avcodec. CodecId(pCodecCtxOrig.GetCodecId())) if pCodec == nil { fmt.Println("Unsupported codec!") os.Exit(1) } // Copy context pCodecCtx := pCodec.AvcodecAllocContext3() if pCodecCtx.AvcodecCopyContext((*avcodec. Context)(unsafe.Pointer(pCodecCtxOrig))) != 0 { fmt.Println("Couldn't copy codec context") os.Exit(1) } // Open codec if pCodecCtx.AvcodecOpen2(pCodec, nil) < 0 { fmt.Println("Could not open codec") os.Exit(1) } // Allocate video frame pFrame := avutil.AvFrameAlloc() // Allocate an AVFrame structure pFrameRGB := avutil.AvFrameAlloc() if pFrameRGB == nil { fmt.Println("Unable to allocate RGB Frame") os.Exit(1) } // Determine required buffer size and allocate buffer numBytes := uintptr(avcodec.AvpictureGetSize( avcodec.AV_PIX_FMT_RGBA, pCodecCtx.Width(), pCodecCtx.Height())) buffer := avutil.AvMalloc(numBytes) // Assign appropriate parts of buffer to image planes in pFrameRGB // Note that pFrameRGB is an AVFrame, but AVFrame is a superset // of AVPicture avp := (*avcodec.Picture)(unsafe.Pointer(pFrameRGB)) avp.AvpictureFill((*uint8)(buffer), avcodec.AV_PIX_FMT_RGBA, pCodecCtx.Width(), pCodecCtx.Height()) // initialize SWS context for software scaling swsCtx := swscale.SwsGetcontext( pCodecCtx.Width(), pCodecCtx.Height(), (swscale.PixelFormat)(pCodecCtx.PixFmt()), pCodecCtx.Width(), pCodecCtx.Height(), avcodec.AV_PIX_FMT_RGBA, avcodec.SWS_BILINEAR, nil, nil, nil, ) // Read frames and save first five frames to disk packet := avcodec.AvPacketAlloc() for pFormatContext.AvReadFrame(packet) >= 0 { // Is this a packet from the video stream? if packet.StreamIndex() == i { // Decode video frame response := pCodecCtx.AvcodecSendPacket(packet) if response < 0 { fmt.Printf("Error while sending a packet to the decoder: %s\n", avutil.ErrorFromCode(response)) } for response >= 0 { response = pCodecCtx.AvcodecReceiveFrame( (*avcodec.Frame)(unsafe.Pointer(pFrame))) if response == avutil.AvErrorEAGAIN || response == avutil.AvErrorEOF { break } else if response < 0 { //fmt.Printf("Error while receiving a frame from the decoder: %s\n", //avutil.ErrorFromCode(response)) //return } // Convert the image from its native format to RGB swscale.SwsScale2(swsCtx, avutil.Data(pFrame), avutil.Linesize(pFrame), 0, pCodecCtx.Height(), avutil.Data(pFrameRGB), avutil.Linesize(pFrameRGB)) // Save the frame to the frame buffer. frame := getFrameRGBA(pFrameRGB, pCodecCtx.Width(), pCodecCtx.Height()) frameBuffer <- frame } } // Free the packet that was allocated by av_read_frame packet.AvFreePacket() } go func() { for { if len(frameBuffer) <= 0 { close(frameBuffer) break } } }() // Free the RGB image avutil.AvFree(buffer) avutil.AvFrameFree(pFrameRGB) // Free the YUV frame avutil.AvFrameFree(pFrame) // Close the codecs pCodecCtx.AvcodecClose() (*avcodec.Context)(unsafe.Pointer(pCodecCtxOrig)).AvcodecClose() // Close the video file pFormatContext.AvformatCloseInput() // Stop after saving frames of first video straem break default: fmt.Println("Didn't find a video stream") os.Exit(1) } } }() return frameBuffer }

We allocate a separate goroutine for the frame decoder so it can do its work in its own pace and just put all the results in the frame buffer.

Converting video frames

Now we need to bring the extracted video frame to the form appropriate for Pixel. First we should extract raw RGBA bytes:

func getFrameRGBA(frame *avutil.Frame, width, height int) *pixel.PictureData { pix := []byte{} for y := 0; y < height; y++ { data0 := avutil.Data(frame)[0] buf := make([]byte, width*4) startPos := uintptr(unsafe.Pointer(data0)) + uintptr(y)*uintptr(avutil.Linesize(frame)[0]) for i := 0; i < width*4; i++ { element := *(*uint8)(unsafe.Pointer(startPos + uintptr(i))) buf[i] = element } pix = append(pix, buf...) } return pixToPictureData(pix, width, height) }

To create *pixel.PictureData out of these bytes, we'll use this simple code:

func pixToPictureData(pixels []byte, width, height int) *pixel.PictureData { picData := pixel.MakePictureData(pixel. R(0, 0, float64(width), float64(height))) for y := height - 1; y >= 0; y-- { for x := 0; x < width; x++ { picData.Pix[(height-y-1)*width+x].R = pixels[y*width*4+x*4+0] picData.Pix[(height-y-1)*width+x].G = pixels[y*width*4+x*4+1] picData.Pix[(height-y-1)*width+x].B = pixels[y*width*4+x*4+2] picData.Pix[(height-y-1)*width+x].A = pixels[y*width*4+x*4+3] } } return picData }

We need to fill the Pix array vice versa because this is how Pixel treats picture data.

Rendering video frames

Now let's create a new animated sprite to output the video frames:

videoSprite := pixel.NewSprite(nil, pixel.Rect{}) videoTransform := pixel.IM.Moved(pixel.V( float64(WindowWidth)/2, float64(WindowHeight)/2)) frameBuffer := readVideoFrames(os.Args[1])

The path to the video file is specified as a command line argument.

Then let's start rendering the video frames:

select { case frame, ok: = <-frameBuffer: if !ok { os.Exit(0) } if frame != nil { videoSprite.Set(frame, frame.Rect) } default: } videoSprite.Draw(win, videoTransform)