K3s

Since v0.21.0

Introduction

The Testcontainers module for K3s.

Adding this module to your project dependencies

Please run the following command to add the K3s module to your Go dependencies:

go get github.com/testcontainers/testcontainers-go/modules/k3s

Usage example

Test for a K3s container

package k3s_test

import (
    "context"
    "fmt"
    "strings"
    "testing"
    "time"

    "github.com/containerd/platforms"
    "github.com/stretchr/testify/require"
    corev1 "k8s.io/api/core/v1"
    metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    kwait "k8s.io/apimachinery/pkg/util/wait"
    "k8s.io/client-go/kubernetes"
    "k8s.io/client-go/tools/clientcmd"

    "github.com/testcontainers/testcontainers-go"
    "github.com/testcontainers/testcontainers-go/modules/k3s"
    "github.com/testcontainers/testcontainers-go/wait"
)

func Test_LoadImages(t *testing.T) {
    // Give up to three minutes to run this test
    ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(3*time.Minute))
    defer cancel()

    k3sContainer, err := k3s.Run(ctx, "rancher/k3s:v1.27.1-k3s1")
    testcontainers.CleanupContainer(t, k3sContainer)
    require.NoError(t, err)

    kubeConfigYaml, err := k3sContainer.GetKubeConfig(ctx)
    require.NoError(t, err)

    restcfg, err := clientcmd.RESTConfigFromKubeConfig(kubeConfigYaml)
    require.NoError(t, err)

    k8s, err := kubernetes.NewForConfig(restcfg)
    require.NoError(t, err)

    provider, err := testcontainers.ProviderDocker.GetProvider()
    require.NoError(t, err)

    // This function only works for single architecture images
    // Forces the test to use a single-arch version of the image
    arch := platforms.DefaultSpec().Architecture
    if platforms.DefaultSpec().Variant != "" {
        arch += platforms.DefaultSpec().Variant
    }
    nginxImg := arch + "/nginx"

    // ensure nginx image is available locally
    err = provider.PullImage(ctx, nginxImg)
    require.NoError(t, err)

    t.Run("Test load image not available", func(t *testing.T) {
        err := k3sContainer.LoadImages(ctx, "fake.registry/fake:non-existing")
        require.Error(t, err)
    })

    t.Run("Test load image in cluster", func(t *testing.T) {
        err := k3sContainer.LoadImages(ctx, nginxImg)
        require.NoError(t, err)

        pod := &corev1.Pod{
            TypeMeta: metav1.TypeMeta{
                Kind:       "Pod",
                APIVersion: "v1",
            },
            ObjectMeta: metav1.ObjectMeta{
                Name: "test-pod",
            },
            Spec: corev1.PodSpec{
                Containers: []corev1.Container{
                    {
                        Name:            "nginx",
                        Image:           nginxImg,
                        ImagePullPolicy: corev1.PullNever, // use image only if already present
                    },
                },
            },
        }

        _, err = k8s.CoreV1().Pods("default").Create(ctx, pod, metav1.CreateOptions{})
        require.NoError(t, err)

        err = kwait.PollUntilContextCancel(ctx, time.Second, true, func(ctx context.Context) (bool, error) {
            state, err := getTestPodState(ctx, k8s)
            if err != nil {
                return false, err
            }
            if state.Terminated != nil {
                return false, fmt.Errorf("pod terminated: %v", state.Terminated)
            }
            return state.Running != nil, nil
        })
        require.NoError(t, err)

        state, err := getTestPodState(ctx, k8s)
        require.NoError(t, err)
        require.NotNil(t, state.Running)
    })
}

func Test_LoadImagesWithPlatform(t *testing.T) {
    // Give up to three minutes to run this test
    ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(3*time.Minute))
    defer cancel()

    k3sContainer, err := k3s.Run(ctx, "rancher/k3s:v1.27.1-k3s1")
    testcontainers.CleanupContainer(t, k3sContainer)
    require.NoError(t, err)

    kubeConfigYaml, err := k3sContainer.GetKubeConfig(ctx)
    require.NoError(t, err)

    restcfg, err := clientcmd.RESTConfigFromKubeConfig(kubeConfigYaml)
    require.NoError(t, err)

    k8s, err := kubernetes.NewForConfig(restcfg)
    require.NoError(t, err)

    provider, err := testcontainers.ProviderDocker.GetProvider()
    require.NoError(t, err)

    // This function only works for single architecture images.
    // Forces the test to use a single-arch version of the image.
    hostPlatform := platforms.DefaultSpec()
    hostPlatform.OS = "linux"
    arch := hostPlatform.Architecture
    if hostPlatform.Variant != "" {
        arch += hostPlatform.Variant
    }
    nginxImg := arch + "/nginx"

    // ensure nginx image is available locally for the host platform
    err = provider.PullImageWithOpts(ctx, nginxImg, testcontainers.PullDockerImageWithPlatform(hostPlatform))
    require.NoError(t, err)

    t.Run("Test load image not available", func(t *testing.T) {
        p, _ := platforms.Parse("linux/amd64")
        err := k3sContainer.LoadImagesWithPlatform(ctx, []string{"fake.registry/fake:non-existing"}, &p)
        require.Error(t, err)
    })

    t.Run("Test load image with wrong architecture", func(t *testing.T) {
        pullPlatform, _ := platforms.Parse("linux/s390x")
        img := "nginx:mainline"
        err = provider.PullImageWithOpts(ctx, img, testcontainers.PullDockerImageWithPlatform(pullPlatform))
        require.NoError(t, err)

        loadPlatform, _ := platforms.Parse("linux/amd64")
        err := k3sContainer.LoadImagesWithPlatform(ctx, []string{img}, &loadPlatform)
        require.Error(t, err)
        requirePlatformMismatchError(t, err, img, platforms.Format(loadPlatform))
    })

    t.Run("Test load image in cluster", func(t *testing.T) {
        err := k3sContainer.LoadImagesWithPlatform(ctx, []string{nginxImg}, &hostPlatform)
        require.NoError(t, err)

        pod := &corev1.Pod{
            TypeMeta: metav1.TypeMeta{
                Kind:       "Pod",
                APIVersion: "v1",
            },
            ObjectMeta: metav1.ObjectMeta{
                Name: "test-pod",
            },
            Spec: corev1.PodSpec{
                Containers: []corev1.Container{
                    {
                        Name:            "nginx",
                        Image:           nginxImg,
                        ImagePullPolicy: corev1.PullNever, // use image only if already present
                    },
                },
            },
        }

        _, err = k8s.CoreV1().Pods("default").Create(ctx, pod, metav1.CreateOptions{})
        require.NoError(t, err)

        err = kwait.PollUntilContextCancel(ctx, time.Second, true, func(ctx context.Context) (bool, error) {
            state, err := getTestPodState(ctx, k8s)
            if err != nil {
                return false, err
            }
            if state.Terminated != nil {
                return false, fmt.Errorf("pod terminated: %v", state.Terminated)
            }
            return state.Running != nil, nil
        })
        require.NoError(t, err)

        state, err := getTestPodState(ctx, k8s)
        require.NoError(t, err)
        require.NotNil(t, state.Running)
    })
}

func requirePlatformMismatchError(t *testing.T, err error, img, platform string) {
    t.Helper()

    errMsg := err.Error()
    legacy := fmt.Sprintf(
        "image with reference %s was found but does not provide the specified platform (%s)",
        img,
        platform,
    )
    modern := "no suitable export target found for platform " + platform

    require.True(t,
        strings.Contains(errMsg, legacy) || strings.Contains(errMsg, modern),
        "unexpected platform mismatch error: %q", errMsg,
    )
}

func getTestPodState(ctx context.Context, k8s *kubernetes.Clientset) (corev1.ContainerState, error) {
    var pod *corev1.Pod
    var err error
    pod, err = k8s.CoreV1().Pods("default").Get(ctx, "test-pod", metav1.GetOptions{})
    if err != nil || len(pod.Status.ContainerStatuses) == 0 {
        return corev1.ContainerState{}, err
    }
    return pod.Status.ContainerStatuses[0].State, nil
}

func Test_APIServerReady(t *testing.T) {
    ctx := context.Background()

    k3sContainer, err := k3s.Run(ctx, "rancher/k3s:v1.27.1-k3s1")
    testcontainers.CleanupContainer(t, k3sContainer)
    require.NoError(t, err)

    kubeConfigYaml, err := k3sContainer.GetKubeConfig(ctx)
    require.NoError(t, err)

    restcfg, err := clientcmd.RESTConfigFromKubeConfig(kubeConfigYaml)
    require.NoError(t, err)

    k8s, err := kubernetes.NewForConfig(restcfg)
    require.NoError(t, err)

    pod := &corev1.Pod{
        TypeMeta: metav1.TypeMeta{
            Kind:       "Pod",
            APIVersion: "v1",
        },
        ObjectMeta: metav1.ObjectMeta{
            Name: "test-pod",
        },
        Spec: corev1.PodSpec{
            Containers: []corev1.Container{
                {
                    Name:  "nginx",
                    Image: "nginx",
                },
            },
        },
    }

    _, err = k8s.CoreV1().Pods("default").Create(context.Background(), pod, metav1.CreateOptions{})
    require.NoError(t, err)
}

func Test_WithManifestOption(t *testing.T) {
    ctx := context.Background()

    k3sContainer, err := k3s.Run(ctx,
        "rancher/k3s:v1.27.1-k3s1",
        k3s.WithManifest("nginx-manifest.yaml"),
        testcontainers.WithAdditionalWaitStrategy(wait.ForExec([]string{"kubectl", "wait", "pod", "nginx", "--for=condition=Ready"})),
    )
    testcontainers.CleanupContainer(t, k3sContainer)
    require.NoError(t, err)
}

Module Reference

Run function

• Since v0.32.0

Info

The RunContainer(ctx, opts...) function is deprecated and will be removed in the next major release of Testcontainers for Go.

The K3s module exposes one entrypoint function to create the K3s container, and this function receives three parameters:

func Run(ctx context.Context, img string, opts ...testcontainers.ContainerCustomizer) (*K3sContainer, error)

• context.Context, the Go context.
• string, the Docker image to use.
• testcontainers.ContainerCustomizer, a variadic argument for passing options.

Container Ports

These are the ports used by the K3s container:

Container Ports

defaultKubeSecurePort     = "6443/tcp"
defaultRancherWebhookPort = "8443/tcp"

Image

Use the second argument in the Run function to set a valid Docker image. In example: Run(context.Background(), "rancher/k3s:v1.27.1-k3s1").

Container Options

When starting the K3s container, you can pass options in a variadic way to configure it.

WithManifest

• Since v0.29.0

The WithManifest option loads a manifest obtained from a local file into the cluster. K3s applies it automatically during the startup process

func WithManifest(manifestPath string) testcontainers.CustomizeRequestOption

Example:

        WithManifest("nginx-manifest.yaml")

The following options are exposed by the testcontainers package.

Basic Options

• WithExposedPorts Since v0.37.0
• WithEnv Since v0.29.0
• WithWaitStrategy Since v0.20.0
• WithAdditionalWaitStrategy Since v0.38.0
• WithWaitStrategyAndDeadline Since v0.20.0
• WithAdditionalWaitStrategyAndDeadline Since v0.38.0
• WithEntrypoint Since v0.37.0
• WithEntrypointArgs Since v0.37.0
• WithCmd Since v0.37.0
• WithCmdArgs Since v0.37.0
• WithLabels Since v0.37.0

Lifecycle Options

• WithLifecycleHooks Since v0.38.0
• WithAdditionalLifecycleHooks Since v0.38.0
• WithStartupCommand Since v0.25.0
• WithAfterReadyCommand Since v0.28.0

Files & Mounts Options

• WithFiles Since v0.37.0
• WithMounts Since v0.37.0
• WithTmpfs Since v0.37.0
• WithImageMount Since v0.37.0

Build Options

• WithDockerfile Since v0.37.0

Logging Options

• WithLogConsumers Since v0.28.0
• WithLogConsumerConfig Since v0.38.0
• WithLogger Since v0.29.0

Image Options

• WithAlwaysPull Since v0.38.0
• WithImageSubstitutors Since v0.26.0
• WithImagePlatform Since v0.38.0

Networking Options

• WithNetwork Since v0.27.0
• WithNetworkByName Since v0.38.0
• WithBridgeNetwork Since v0.38.0
• WithNewNetwork Since v0.27.0

Advanced Options

• WithHostPortAccess Since v0.31.0
• WithConfigModifier Since v0.20.0
• WithHostConfigModifier Since v0.20.0
• WithEndpointSettingsModifier Since v0.20.0
• CustomizeRequest Since v0.20.0
• WithName Since v0.38.0
• WithNoStart Since v0.38.0
• WithProvider Since v0.39.0

Experimental Options

• WithReuseByName Since v0.37.0

Container Methods

The K3s container exposes the following methods:

GetKubeConfig

• Since v0.21.0

The GetKubeConfig method returns the K3s cluster's kubeconfig, including the server URL, to be used for connecting to the Kubernetes Rest Client API using a Kubernetes client. It'll be returned in the format of []bytes.

Get KubeConfig

package k3s_test

import (
    "context"
    "fmt"
    "log"

    v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    "k8s.io/client-go/kubernetes"
    "k8s.io/client-go/tools/clientcmd"

    "github.com/testcontainers/testcontainers-go"
    "github.com/testcontainers/testcontainers-go/modules/k3s"
)

func ExampleRun() {
    // runK3sContainer {
    ctx := context.Background()

    k3sContainer, err := k3s.Run(ctx, "rancher/k3s:v1.27.1-k3s1")
    defer func() {
        if err := testcontainers.TerminateContainer(k3sContainer); err != nil {
            log.Printf("failed to terminate container: %s", err)
        }
    }()
    if err != nil {
        log.Printf("failed to start container: %s", err)
        return
    }
    // }

    state, err := k3sContainer.State(ctx)
    if err != nil {
        log.Printf("failed to get container state: %s", err)
        return
    }

    fmt.Println(state.Running)

    kubeConfigYaml, err := k3sContainer.GetKubeConfig(ctx)
    if err != nil {
        log.Printf("failed to get kubeconfig: %s", err)
        return
    }

    restcfg, err := clientcmd.RESTConfigFromKubeConfig(kubeConfigYaml)
    if err != nil {
        log.Printf("failed to create rest config: %s", err)
        return
    }

    k8s, err := kubernetes.NewForConfig(restcfg)
    if err != nil {
        log.Printf("failed to create k8s client: %s", err)
        return
    }

    nodes, err := k8s.CoreV1().Nodes().List(ctx, v1.ListOptions{})
    if err != nil {
        log.Printf("failed to list nodes: %s", err)
        return
    }

    fmt.Println(len(nodes.Items))

    // Output:
    // true
    // 1
}

LoadImages

• Since v0.25.0

The LoadImages method imports images from the local Docker daemon into the k3s cluster and makes them available to pods.

func (c *K3sContainer) LoadImages(ctx context.Context, images ...string) error

This is useful for testing images built locally without pushing them to a registry or configuring k3s to use a private registry.

Images must already be present on the Docker host running the test. DockerProvider.PullImage is enough for single-architecture image references. When you need a specific OCI platform, use DockerProvider.PullImageWithOpts with PullDockerImageWithPlatform.

LoadImages delegates to LoadImagesWithOpts without save options. It works best with single-architecture image references (for example amd64/nginx). For multi-architecture images, use LoadImagesWithPlatform or LoadImagesWithOpts.

When creating pods that use loaded images, set imagePullPolicy: Never so Kubernetes uses the imported image instead of pulling from a registry.

Example:

provider, err := testcontainers.ProviderDocker.GetProvider()
if err != nil {
    // handle error
}

if err := provider.PullImage(ctx, "amd64/nginx"); err != nil {
    // handle error
}

if err := k3sContainer.LoadImages(ctx, "amd64/nginx"); err != nil {
    // handle error
}

LoadImagesWithPlatform

The LoadImagesWithPlatform method is a convenience wrapper around LoadImagesWithOpts that exports and imports an image for a specific OCI platform.

func (c *K3sContainer) LoadImagesWithPlatform(ctx context.Context, images []string, platform *v1.Platform) error

When platform is nil, behaviour matches LoadImages. When platform is set, the image is exported and imported for that OCI platform.

Use this method on multi-architecture hosts or when loading multi-architecture image tags. Pull the same platform into Docker first, then load it into k3s:

hostPlatform := platforms.DefaultSpec()
hostPlatform.OS = "linux"

provider, err := testcontainers.ProviderDocker.GetProvider()
if err != nil {
    // handle error
}

if err := provider.PullImageWithOpts(
    ctx,
    "amd64/nginx",
    testcontainers.PullDockerImageWithPlatform(hostPlatform),
); err != nil {
    // handle error
}

if err := k3sContainer.LoadImagesWithPlatform(ctx, []string{"amd64/nginx"}, &hostPlatform); err != nil {
    // handle error
}

LoadImagesWithOpts

• Since v0.25.0

The LoadImagesWithOpts method imports local images into the k3s cluster, passing SaveImageOption values through to docker save.

func (c *K3sContainer) LoadImagesWithOpts(ctx context.Context, images []string, opts ...testcontainers.SaveImageOption) error

When SaveDockerImageWithPlatforms is passed, containerd import uses the same platform. Without platform options, import does not use --all-platforms.

For multiple images on different architectures, call LoadImagesWithOpts once per image and platform.

Example with platform:

hostPlatform := platforms.DefaultSpec()
hostPlatform.OS = "linux"

provider, err := testcontainers.ProviderDocker.GetProvider()
if err != nil {
    // handle error
}

if err := provider.PullImageWithOpts(
    ctx,
    "amd64/nginx",
    testcontainers.PullDockerImageWithPlatform(hostPlatform),
); err != nil {
    // handle error
}

if err := k3sContainer.LoadImagesWithOpts(
    ctx,
    []string{"amd64/nginx"},
    testcontainers.SaveDockerImageWithPlatforms(hostPlatform),
); err != nil {
    // handle error
}