If we can't create a container because one aleady exists with the same name, kill the existing container and return. Check CPU sets in when container start fails. Add more logging.
@@ -348,6 +348,20 @@ func (p *dockerProvider) Start(ctx gocontext.Context, startAttributes *StartAttr
}
containerName := hostnameFromContext(ctx)
+ existingContainer, err := p.client.ContainerInspect(ctx, containerName)
+ if err == nil {
+ err := p.client.ContainerRemove(ctx, existingContainer.ID,
+ dockertypes.ContainerRemoveOptions{
+ Force: true,
+ RemoveLinks: false,
+ RemoveVolumes: true,
+ })
+ if err != nil {
+ logger.WithField("err", err).Error("couldn't remove preexisting container before create")
+ } else {
+ logger.Warn("removed preexisting container before create")
+ }
+ }
dockerConfig := &dockercontainer.Config{
Cmd: p.runCmd,
@@ -390,7 +404,7 @@ func (p *dockerProvider) Start(ctx gocontext.Context, startAttributes *StartAttr
if err != nil {
logger.WithField("err", err).Error("couldn't create container")
- defer p.checkinCPUSets(ctx, cpuSets)
+ p.checkinCPUSets(ctx, cpuSets)
err := p.client.ContainerRemove(ctx, container.ID,
dockertypes.ContainerRemoveOptions{
@@ -409,7 +423,8 @@ func (p *dockerProvider) Start(ctx gocontext.Context, startAttributes *StartAttr
err = p.client.ContainerStart(ctx, container.ID, dockertypes.ContainerStartOptions{})
if err != nil {
- defer p.checkinCPUSets(ctx, cpuSets)
+ logger.WithField("err", err).Error("couldn't start container")
+ p.checkinCPUSets(ctx, cpuSets)
return nil, err
}
@@ -445,6 +460,7 @@ func (p *dockerProvider) Start(ctx gocontext.Context, startAttributes *StartAttr
return nil, err
case <-ctx.Done():
if ctx.Err() == gocontext.DeadlineExceeded {
+ p.checkinCPUSets(ctx, cpuSets)
metrics.Mark("worker.vm.provider.docker.boot.timeout")
}
return nil, ctx.Err()
@@ -506,8 +522,8 @@ func (p *dockerProvider) checkinCPUSets(ctx gocontext.Context, sets string) {
}
p.cpuSets[int(cpu)] = false
- logger.WithField("cpu_sets", sets).Info("checked in")
}
+ logger.WithField("cpu_sets", sets).Info("checked in")
}
func (i *dockerInstance) sshConnection(ctx gocontext.Context) (ssh.Connection, error) {
@@ -654,10 +670,12 @@ func (i *dockerInstance) runScriptSSH(ctx gocontext.Context, output io.Writer) (
func (i *dockerInstance) Stop(ctx gocontext.Context) error {
defer i.provider.checkinCPUSets(ctx, i.container.HostConfig.Resources.CpusetCpus)
+ logger := context.LoggerFromContext(ctx).WithField("self", "backend/docker_provider")
timeout := 30 * time.Second
err := i.client.ContainerStop(ctx, i.container.ID, &timeout)
if err != nil {
+ logger.Warn("couldn't stop container")
return err
}
