knot knot

CephFS

The CephFS driver mounts a CephFS filesystem into the container, making it ideal for shared storage between multiple containers or servers.

CephFS Controller

Below is the Nomad job specification for deploying the CephFS controller:

csi-cephfs-controller.hcl 
job "csi-cephfs-controller" {
  datacenters = ["dc1"]
  priority = 100

  group "controller" {
    network {
      port "metrics" {}
    }
    task "ceph-controller" {
      template {
        data        = <<EOF
[{
    "clusterID": "def7c9bb-f0a8-429b-b55a-386c2335d92d",
    "monitors": [
        "192.168.0.100",
        "192.168.0.101",
        "192.168.0.102"
    ]
}]
EOF
        destination = "local/config.json"
        change_mode = "restart"
      }

      driver = "docker"
      config {
        image = "quay.io/cephcsi/cephcsi:latest"
        mount {
          type = "bind"
          source = "local/config.json"
          target = "/etc/ceph-csi-config/config.json"
        }
        mount {
          type     = "tmpfs"
          target   = "/tmp/csi/keys"
          readonly = false
          tmpfs_options = {
            size = 1000000 # size in bytes
          }
        }
        args = [
          "--type=cephfs",
          "--controllerserver=true",
          "--drivername=cephfs.csi.ceph.com",
          "--endpoint=unix://csi/csi.sock",
          "--nodeid=${node.unique.name}",
          "--instanceid=${node.unique.name}-controller",
          "--pidlimit=-1",
          "--logtostderr=true",
          "--v=5",
          "--metricsport=$${NOMAD_PORT_metrics}"
        ]
      }
      resources {
        cpu    = 10
        memory = 32
      }
      service {
        name = "${NOMAD_JOB_NAME}"
        port = "metrics"
        tags = [ "prometheus" ]
      }
      csi_plugin {
        id        = "cephfs"
        type      = "controller"
        mount_dir = "/csi"
      }
    }
  }
}

CephFS Node

Below is the Nomad job specification for deploying the CephFS node:

csi-cephfs-node.hcl 
job "csi-cephfs-node" {
  datacenters = ["dc1"]
  type = "system"
  priority = 100

  group "node" {
    network {
      port "metrics" {}
    }
    task "ceph-node" {
      driver = "docker"
      template {
        data        = <<EOF
[{
    "clusterID": "def7c9bb-f0a8-429b-b55a-386c2335d92d",
    "monitors": [
        "192.168.0.100",
        "192.168.0.101",
        "192.168.0.102"
    ]
}]
EOF
        destination = "local/config.json"
        change_mode = "restart"
      }
      config {
        image = "quay.io/cephcsi/cephcsi:latest"
        mount {
          type = "bind"
          source = "local/config.json"
          target = "/etc/ceph-csi-config/config.json"
        }
        mount {
          type = "bind"
          source = "/lib/modules/${attr.kernel.version}"
          target = "/lib/modules/${attr.kernel.version}"
        }
        mount {
          type     = "tmpfs"
          target   = "/tmp/csi/keys"
          readonly = false
          tmpfs_options = {
            size = 1000000 # size in bytes
          }
        }

        args = [
          "--type=cephfs",
          "--drivername=cephfs.csi.ceph.com",
          "--nodeserver=true",
          "--endpoint=unix://csi/csi.sock",
          "--nodeid=${node.unique.name}",
          "--instanceid=${node.unique.name}-nodes",
          "--pidlimit=-1",
          "--logtostderr=true",
          "--v=5",
          "--metricsport=$${NOMAD_PORT_metrics}"
        ]
        privileged = true
      }
      resources {
        cpu    = 10
        memory = 64
      }
      service {
        name = "${NOMAD_JOB_NAME}"
        port = "metrics"
        tags = [ "prometheus" ]
      }
      csi_plugin {
        id        = "cephfs"
        type      = "node"
        mount_dir = "/csi"
      }
    }
  }
}

Example Usage in Knot

Below is an example of how to use the CephFS driver. This configuration can be added to the Volumes section of a template or as a standalone volume:

volumes:
  - id: "test-volume"
    name: "test-volume"
    plugin_id: "cephfs"
    capacity_min: 10G
    capacity_max: 10G
    capabilities:
      - access_mode: "multi-node-multi-writer"
        attachment_mode: "file-system"
    secrets:
      adminID: "nomad.cephfs"
      adminKey: "FWrwe2323d3QWdqwdqwdce!@+23fiseQW/DQWd=="
      userID: "nomad.cephfs"
      userKey: "FWrwe2323d3QWdqwdqwdce!@+23fiseQW/DQWd=="
    parameters:
      clusterID: "b13bf70b-253c-4691-b3f2-46df86f4318c"
      fsName: "cephfs"

Example Usage in a Nomad Job

The volume is referenced from a job using a volume stanza with type = "csi", where attachment_mode and access_mode must match the volume’s capabilities. It is then mounted into the task with a volume_mount stanza:

job "${{.user.username}}-${{.space.name}}" {
  datacenters = ["dc1"]

  update {
    max_parallel     = 1
    min_healthy_time = "30s"
    healthy_deadline = "1m"
    auto_revert      = false
  }

  group "example" {
    count = 1

    volume "data_volume" {
      type            = "csi"
      source          = "test-volume"
      read_only       = false
      attachment_mode = "file-system"
      access_mode     = "multi-node-multi-writer"
    }

    task "example" {
      driver = "docker"
      config {
        image    = "paularlott/knot-ubuntu:24.04"
        hostname = "${{ .space.name }}"
      }

      env {
        KNOT_SERVER         = "${{.server.url}}"
        KNOT_AGENT_ENDPOINT = "${{.server.agent_endpoint}}"
        KNOT_SPACEID        = "${{.space.id}}"
        KNOT_USER           = "${{.user.username}}"
        TZ                  = "${{.user.timezone}}"
      }

      volume_mount {
        volume      = "data_volume"
        destination = "/data"
      }

      resources {
        cores  = 1
        memory = 512
      }
    }
  }
}