Embedding anything in a golang binary

What is it about?

One good thing about golang is that you compile everything to a binary. So easy to deploy, so easy to manage, so small.

But what happens when you need to embed files in there? Migration files, static files, whatever you can think of?

Well, you can simply deploy it in a container, add this files, you are going to say, and you are probably right, that’s imho, the cleanest solution. But sometimes, you can’t, sometimes you can’t deploy containers and you can’t add files next to your binary.

Choosing the library

There are multiple solutions out there and a great article has done a great work at comparing most of them.

I decided to follow the author advice and go for vfsgen. Why? Mostly because it can be used without any stand alone executable and that’s pretty convenient.

Implementation

Defining what and where

First, let’s choose the file we want to turn into a binary.

mkdir -p wow/embed
echo "lerooyyyy" > wow/embed/jenkins.txt

Then, let’s create the file wow/wow_generate.go that will generate the binaries, in our case, the http.FileSystem:

    // +build ignore

    package main

    import (
        "log"
        "net/http"

        "github.com/shurcooL/vfsgen"
    )

    func main() {
        err := vfsgen.Generate(
            http.Dir("./embed"),
            vfsgen.Options{
                Filename:     "./wow_vfsdata.go",
                PackageName:  "wow",
                VariableName: "noobAssets",
            })
        if err != nil {
            log.Fatalln(err)
        }
    }

Let’s quickly explain the details:

// +build ignore simply means that this file will be ignored when we will build our go program.

the first argument to vfsgen.Generate, http.Dir("./embed"), is the folder you want to embed.

the options are pretty obvious. Note that you can choose to use a global variable name, to have access to it outside of the wow package, but I think it’s probably not a good practice.

Generate

Now, how to make it happen? Use go generate!

Create a file, wow/wow.go, in the same folder you created the previous one:

package wow

//go:generate go run wow_generate.go

then, here we are

go generate ./wow

and you can now see wow_vfsdata.go created in the folder.

Use

The file created is a http.FileSystem, you can mount it to serve it with http.FileServer(noobAssets) but you can also turn it into a more classical FileSystem with a little bit of code, we’ll come back to it later, enjoy your everything binary now!

Here is the code created:

    // Code generated by vfsgen; DO NOT EDIT.

    package wow

    import (
        "bytes"
        "compress/gzip"
        "fmt"
        "io"
        "io/ioutil"
        "net/http"
        "os"
        pathpkg "path"
        "time"
    )

    // noobAssets statically implements the virtual filesystem provided to vfsgen.
    var noobAssets = func() http.FileSystem {
        fs := vfsgen۰FS{
            "/": &vfsgen۰DirInfo{
                name:    "/",
                modTime: time.Date(2018, 9, 29, 3, 12, 35, 780695280, time.UTC),
            },
            "/jenkins.txt": &vfsgen۰FileInfo{
                name:    "jenkins.txt",
                modTime: time.Date(2018, 9, 29, 3, 12, 35, 780787937, time.UTC),
                content: []byte("\x6c\x65\x72\x6f\x6f\x79\x79\x79\x79\x0a"),
            },
        }
        fs["/"].(*vfsgen۰DirInfo).entries = []os.FileInfo{
            fs["/jenkins.txt"].(os.FileInfo),
        }

        return fs
    }()

    type vfsgen۰FS map[string]interface{}

    func (fs vfsgen۰FS) Open(path string) (http.File, error) {
        path = pathpkg.Clean("/" + path)
        f, ok := fs[path]
        if !ok {
            return nil, &os.PathError{Op: "open", Path: path, Err: os.ErrNotExist}
        }

        switch f := f.(type) {
        case *vfsgen۰FileInfo:
            return &vfsgen۰File{
                vfsgen۰FileInfo: f,
                Reader:          bytes.NewReader(f.content),
            }, nil
        case *vfsgen۰DirInfo:
            return &vfsgen۰Dir{
                vfsgen۰DirInfo: f,
            }, nil
        default:
            // This should never happen because we generate only the above types.
            panic(fmt.Sprintf("unexpected type %T", f))
        }
    }

    // We already imported "compress/gzip" and "io/ioutil", but ended up not using them. Avoid unused import error.
    var _ = gzip.Reader{}
    var _ = ioutil.Discard

    // vfsgen۰FileInfo is a static definition of an uncompressed file (because it's not worth gzip compressing).
    type vfsgen۰FileInfo struct {
        name    string
        modTime time.Time
        content []byte
    }

    func (f *vfsgen۰FileInfo) Readdir(count int) ([]os.FileInfo, error) {
        return nil, fmt.Errorf("cannot Readdir from file %s", f.name)
    }
    func (f *vfsgen۰FileInfo) Stat() (os.FileInfo, error) { return f, nil }

    func (f *vfsgen۰FileInfo) NotWorthGzipCompressing() {}

    func (f *vfsgen۰FileInfo) Name() string       { return f.name }
    func (f *vfsgen۰FileInfo) Size() int64        { return int64(len(f.content)) }
    func (f *vfsgen۰FileInfo) Mode() os.FileMode  { return 0444 }
    func (f *vfsgen۰FileInfo) ModTime() time.Time { return f.modTime }
    func (f *vfsgen۰FileInfo) IsDir() bool        { return false }
    func (f *vfsgen۰FileInfo) Sys() interface{}   { return nil }

    // vfsgen۰File is an opened file instance.
    type vfsgen۰File struct {
        *vfsgen۰FileInfo
        *bytes.Reader
    }

    func (f *vfsgen۰File) Close() error {
        return nil
    }

    // vfsgen۰DirInfo is a static definition of a directory.
    type vfsgen۰DirInfo struct {
        name    string
        modTime time.Time
        entries []os.FileInfo
    }

    func (d *vfsgen۰DirInfo) Read([]byte) (int, error) {
        return 0, fmt.Errorf("cannot Read from directory %s", d.name)
    }
    func (d *vfsgen۰DirInfo) Close() error               { return nil }
    func (d *vfsgen۰DirInfo) Stat() (os.FileInfo, error) { return d, nil }

    func (d *vfsgen۰DirInfo) Name() string       { return d.name }
    func (d *vfsgen۰DirInfo) Size() int64        { return 0 }
    func (d *vfsgen۰DirInfo) Mode() os.FileMode  { return 0755 | os.ModeDir }
    func (d *vfsgen۰DirInfo) ModTime() time.Time { return d.modTime }
    func (d *vfsgen۰DirInfo) IsDir() bool        { return true }
    func (d *vfsgen۰DirInfo) Sys() interface{}   { return nil }

    // vfsgen۰Dir is an opened dir instance.
    type vfsgen۰Dir struct {
        *vfsgen۰DirInfo
        pos int // Position within entries for Seek and Readdir.
    }

    func (d *vfsgen۰Dir) Seek(offset int64, whence int) (int64, error) {
        if offset == 0 && whence == io.SeekStart {
            d.pos = 0
            return 0, nil
        }
        return 0, fmt.Errorf("unsupported Seek in directory %s", d.name)
    }

    func (d *vfsgen۰Dir) Readdir(count int) ([]os.FileInfo, error) {
        if d.pos >= len(d.entries) && count > 0 {
            return nil, io.EOF
        }
        if count <= 0 || count > len(d.entries)-d.pos {
            count = len(d.entries) - d.pos
        }
        e := d.entries[d.pos : d.pos+count]
        d.pos += count
        return e, nil
    }