Runc: libcontainer: executing multiple processes in the same container in parallel fails

[LittleLightLittleFire]

Tested on the master branch commit efb2bc3. Kernel: 4.10.11-1-ARCH

The following code uses libcontainer to create N number of echo programs in the same container.
The processes fails to execute with the following error output:

WARN[0000] os: process already finished                 
2017/05/02 20:42:53 0 container_linux.go:262: starting container process caused "process_linux.go:81: executing setns process caused \"exit status 15\""
WARN[0000] os: process already finished                 
2017/05/02 20:42:53 1 container_linux.go:262: starting container process caused "process_linux.go:81: executing setns process caused \"exit status 15\""
WARN[0000] os: process already finished                 
2017/05/02 20:42:53 2 container_linux.go:262: starting container process caused "process_linux.go:81: executing setns process caused \"exit status 15\""
WARN[0000] os: process already finished                 
2017/05/02 20:42:53 3 container_linux.go:262: starting container process caused "process_linux.go:81: executing setns process caused \"exit status 15\""
WARN[0000] os: process already finished                 
2017/05/02 20:42:53 4 container_linux.go:262: starting container process caused "process_linux.go:81: executing setns process caused \"exit status 15\""
WARN[0000] os: process already finished                 
2017/05/02 20:42:53 5 container_linux.go:262: starting container process caused "process_linux.go:81: executing setns process caused \"exit status 15\""
WARN[0000] os: process already finished                 
2017/05/02 20:42:53 6 container_linux.go:262: starting container process caused "process_linux.go:81: executing setns process caused \"exit status 15\""
WARN[0000] os: process already finished                 
2017/05/02 20:42:53 7 container_linux.go:262: starting container process caused "process_linux.go:81: executing setns process caused \"exit status 15\""
WARN[0000] os: process already finished                 
2017/05/02 20:42:53 8 container_linux.go:262: starting container process caused "process_linux.go:81: executing setns process caused \"exit status 15\""
2017/05/02 20:42:53 9 <nil>

The failing line is here:
https://github.com/opencontainers/runc/blob/master/libcontainer/nsenter/nsexec.c#L407

The namespace path /proc/[init process]/ns/[namepace] doesn't exist. I suspect this is a race condition.

Code:

package main

import (
    "io/ioutil"
    "log"
    "os"
    "path"
    "runtime"
    "syscall"

    "github.com/opencontainers/runc/libcontainer"
    "github.com/opencontainers/runc/libcontainer/configs"
    _ "github.com/opencontainers/runc/libcontainer/nsenter"
)

const (
    // **** Edit to your image path ****
    imagePath = "/edit/to/your/rootfs"

    runCFolder = "/tmp/runc-test"
    maxMemory  = 256 * 1024 * 1024
    n          = 10
)

func init() {
    if len(os.Args) > 1 && os.Args[1] == "init" {
        runtime.GOMAXPROCS(1)
        runtime.LockOSThread()
        factory, err := libcontainer.New("")
        if err != nil {
            log.Fatal(err)
        }

        if err := factory.StartInitialization(); err != nil {
            log.Fatal(err)
        }
        panic("--this line should have never been executed, congratulations--")
    }
}

func main() {
    cl, err := libcontainer.New(runCFolder, libcontainer.Cgroupfs, libcontainer.InitArgs(os.Args[0], "init"))
    if err != nil {
        log.Fatal(err)
    }

        { // Clean up
                // Delete all running containers
                items, err := ioutil.ReadDir(runCFolder)
                if err != nil {
                        log.Fatal(err)
                }

                for _, item := range items {
                        if item.IsDir() {
                                c, err := cl.Load(item.Name())
                                if err != nil {
                                        _ = os.RemoveAll(path.Join(runCFolder, item.Name()))
                                        continue
                                }

                                if err := c.Destroy(); err != nil {
                                        log.Println("Failed to destroy container:", err)
                                }
                        }
                }
        }

    defaultMountFlags := syscall.MS_NOEXEC | syscall.MS_NOSUID | syscall.MS_NODEV
    container, err := cl.Create("test-container", &configs.Config{
        NoPivotRoot:       false,
        ParentDeathSignal: 9,
        Rootfs:            imagePath,
        Readonlyfs:        true,
        RootPropagation:   0,
        Mounts: []*configs.Mount{
            {
                Source:      "proc",
                Destination: "/proc",
                Device:      "proc",
                Flags:       defaultMountFlags,
            },
            {
                Source:      "tmpfs",
                Destination: "/dev",
                Device:      "tmpfs",
                Flags:       syscall.MS_NOSUID | syscall.MS_STRICTATIME,
                Data:        "mode=755",
            },
            {
                Source:      "devpts",
                Destination: "/dev/pts",
                Device:      "devpts",
                Flags:       syscall.MS_NOSUID | syscall.MS_NOEXEC,
                Data:        "newinstance,ptmxmode=0666,mode=0620,gid=5",
            },
            {
                Source:      "mqueue",
                Destination: "/dev/mqueue",
                Device:      "mqueue",
                Flags:       defaultMountFlags,
            },
            {
                Source:      "sysfs",
                Destination: "/sys",
                Device:      "sysfs",
                Flags:       defaultMountFlags | syscall.MS_RDONLY,
            },
        },
        Devices:    configs.DefaultAutoCreatedDevices,
        Namespaces: []configs.Namespace{
            {Type: configs.NEWNS},
            {Type: configs.NEWUTS},
            {Type: configs.NEWIPC},
            {Type: configs.NEWPID},
            {Type: configs.NEWNET},
        },
        Networks: []*configs.Network{
            {
                Type:    "loopback",
                Address: "127.0.0.1/0",
                Gateway: "localhost",
            },
        },

        Routes: nil,
        Cgroups: &configs.Cgroup{
            Name:   "test-container",
            Parent: "system",
            Resources: &configs.Resources{
                AllowedDevices:   configs.DefaultAllowedDevices,
            },
        },

        Rootless: false,
    })
    if err != nil {
        log.Fatal(err)
    }
    defer container.Destroy()

    resultChan := make(chan error, n)

    // Spawn n echos in parallel
    for i := 0; i < n; i++ {
        go func() (err error) {
            defer func() {
                resultChan <- err
            }()

            p := &libcontainer.Process{
                Args: []string{"/usr/bin/echo"},
            }

            if err := container.Run(p); err != nil {
                return err
            }

            _, err = p.Wait()
            return err
        }()
    }

    for i := 0; i < n; i++ {
        log.Println(i, <-resultChan)
    }
}

[slimsymphony]

@LittleLightLittleFire I met the same issue now, could you please share the solution?