luxolus
/
open-nomad
2
0
Fork 0
Code Issues 1 Pull Requests Packages Releases Wiki Activity

client: Add option to enable hairpinMode on Nomad bridge (#15961) 

* Add `bridge_network_hairpin_mode` client config setting
* Add node attribute: `nomad.bridge.hairpin_mode`
* Changed format string to use `%q` to escape user provided data
* Add test to validate template JSON for developer safety

Co-authored-by: Daniel Bennett <dbennett@hashicorp.com>
This commit is contained in:
Charlie Voiselle 2023-02-02 10:12:15 -05:00 committed by GitHub
parent 37834dffda
commit 4caac1a92f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 184 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.changelog/15961.txt Normal file
View File

@ -0,0 +1,3 @@
```release-note:improvement
client: Add option to enable hairpinMode on Nomad bridge
```

180
client/allocrunner/alloc_runner_test.go
View File

@ -524,12 +524,12 @@ func TestAllocRunner_Lifecycle_Restart(t *testing.T) {
return ar.RestartAll(ev)
},
expectedAfter: map[string]structs.TaskState{
"main": structs.TaskState{State: "running", Restarts: 1},
"prestart-oneshot": structs.TaskState{State: "dead", Restarts: 1},
"prestart-sidecar": structs.TaskState{State: "running", Restarts: 1},
"poststart-oneshot": structs.TaskState{State: "dead", Restarts: 1},
"poststart-sidecar": structs.TaskState{State: "running", Restarts: 1},
"poststop": structs.TaskState{State: "pending", Restarts: 0},
"main": {State: "running", Restarts: 1},
"prestart-oneshot": {State: "dead", Restarts: 1},
"prestart-sidecar": {State: "running", Restarts: 1},
"poststart-oneshot": {State: "dead", Restarts: 1},
"poststart-sidecar": {State: "running", Restarts: 1},
"poststop": {State: "pending", Restarts: 0},
},
},
{
@ -538,12 +538,12 @@ func TestAllocRunner_Lifecycle_Restart(t *testing.T) {
return ar.RestartRunning(ev)
},
expectedAfter: map[string]structs.TaskState{
"main": structs.TaskState{State: "running", Restarts: 1},
"prestart-oneshot": structs.TaskState{State: "dead", Restarts: 0},
"prestart-sidecar": structs.TaskState{State: "running", Restarts: 1},
"poststart-oneshot": structs.TaskState{State: "dead", Restarts: 0},
"poststart-sidecar": structs.TaskState{State: "running", Restarts: 1},
"poststop": structs.TaskState{State: "pending", Restarts: 0},
"main": {State: "running", Restarts: 1},
"prestart-oneshot": {State: "dead", Restarts: 0},
"prestart-sidecar": {State: "running", Restarts: 1},
"poststart-oneshot": {State: "dead", Restarts: 0},
"poststart-sidecar": {State: "running", Restarts: 1},
"poststop": {State: "pending", Restarts: 0},
},
},
{
@ -561,12 +561,12 @@ func TestAllocRunner_Lifecycle_Restart(t *testing.T) {
return ar.RestartAll(ev)
},
expectedAfter: map[string]structs.TaskState{
"main": structs.TaskState{State: "running", Restarts: 1},
"prestart-oneshot": structs.TaskState{State: "dead", Restarts: 1},
"prestart-sidecar": structs.TaskState{State: "running", Restarts: 1},
"poststart-oneshot": structs.TaskState{State: "dead", Restarts: 1},
"poststart-sidecar": structs.TaskState{State: "running", Restarts: 1},
"poststop": structs.TaskState{State: "pending", Restarts: 0},
"main": {State: "running", Restarts: 1},
"prestart-oneshot": {State: "dead", Restarts: 1},
"prestart-sidecar": {State: "running", Restarts: 1},
"poststart-oneshot": {State: "dead", Restarts: 1},
"poststart-sidecar": {State: "running", Restarts: 1},
"poststop": {State: "pending", Restarts: 0},
},
},
{
@ -584,12 +584,12 @@ func TestAllocRunner_Lifecycle_Restart(t *testing.T) {
return ar.RestartRunning(ev)
},
expectedAfter: map[string]structs.TaskState{
"main": structs.TaskState{State: "running", Restarts: 1},
"prestart-oneshot": structs.TaskState{State: "dead", Restarts: 0},
"prestart-sidecar": structs.TaskState{State: "running", Restarts: 1},
"poststart-oneshot": structs.TaskState{State: "dead", Restarts: 0},
"poststart-sidecar": structs.TaskState{State: "running", Restarts: 1},
"poststop": structs.TaskState{State: "pending", Restarts: 0},
"main": {State: "running", Restarts: 1},
"prestart-oneshot": {State: "dead", Restarts: 0},
"prestart-sidecar": {State: "running", Restarts: 1},
"poststart-oneshot": {State: "dead", Restarts: 0},
"poststart-sidecar": {State: "running", Restarts: 1},
"poststop": {State: "pending", Restarts: 0},
},
},
{
@ -599,12 +599,12 @@ func TestAllocRunner_Lifecycle_Restart(t *testing.T) {
return ar.RestartAll(ev)
},
expectedAfter: map[string]structs.TaskState{
"main": structs.TaskState{State: "running", Restarts: 1},
"prestart-oneshot": structs.TaskState{State: "dead", Restarts: 1},
"prestart-sidecar": structs.TaskState{State: "running", Restarts: 1},
"poststart-oneshot": structs.TaskState{State: "dead", Restarts: 1},
"poststart-sidecar": structs.TaskState{State: "running", Restarts: 1},
"poststop": structs.TaskState{State: "pending", Restarts: 0},
"main": {State: "running", Restarts: 1},
"prestart-oneshot": {State: "dead", Restarts: 1},
"prestart-sidecar": {State: "running", Restarts: 1},
"poststart-oneshot": {State: "dead", Restarts: 1},
"poststart-sidecar": {State: "running", Restarts: 1},
"poststop": {State: "pending", Restarts: 0},
},
},
{
@ -616,12 +616,12 @@ func TestAllocRunner_Lifecycle_Restart(t *testing.T) {
return nil
},
expectedAfter: map[string]structs.TaskState{
"main": structs.TaskState{State: "dead", Restarts: 0},
"prestart-oneshot": structs.TaskState{State: "dead", Restarts: 0},
"prestart-sidecar": structs.TaskState{State: "dead", Restarts: 0},
"poststart-oneshot": structs.TaskState{State: "dead", Restarts: 0},
"poststart-sidecar": structs.TaskState{State: "dead", Restarts: 0},
"poststop": structs.TaskState{State: "dead", Restarts: 0},
"main": {State: "dead", Restarts: 0},
"prestart-oneshot": {State: "dead", Restarts: 0},
"prestart-sidecar": {State: "dead", Restarts: 0},
"poststart-oneshot": {State: "dead", Restarts: 0},
"poststart-sidecar": {State: "dead", Restarts: 0},
"poststop": {State: "dead", Restarts: 0},
},
},
{
@ -630,12 +630,12 @@ func TestAllocRunner_Lifecycle_Restart(t *testing.T) {
return ar.RestartTask("main", ev)
},
expectedAfter: map[string]structs.TaskState{
"main": structs.TaskState{State: "running", Restarts: 1},
"prestart-oneshot": structs.TaskState{State: "dead", Restarts: 0},
"prestart-sidecar": structs.TaskState{State: "running", Restarts: 0},
"poststart-oneshot": structs.TaskState{State: "dead", Restarts: 0},
"poststart-sidecar": structs.TaskState{State: "running", Restarts: 0},
"poststop": structs.TaskState{State: "pending", Restarts: 0},
"main": {State: "running", Restarts: 1},
"prestart-oneshot": {State: "dead", Restarts: 0},
"prestart-sidecar": {State: "running", Restarts: 0},
"poststart-oneshot": {State: "dead", Restarts: 0},
"poststart-sidecar": {State: "running", Restarts: 0},
"poststop": {State: "pending", Restarts: 0},
},
},
{
@ -645,12 +645,12 @@ func TestAllocRunner_Lifecycle_Restart(t *testing.T) {
return ar.RestartTask("main", ev)
},
expectedAfter: map[string]structs.TaskState{
"main": structs.TaskState{State: "running", Restarts: 1},
"prestart-oneshot": structs.TaskState{State: "dead", Restarts: 0},
"prestart-sidecar": structs.TaskState{State: "running", Restarts: 0},
"poststart-oneshot": structs.TaskState{State: "dead", Restarts: 0},
"poststart-sidecar": structs.TaskState{State: "running", Restarts: 0},
"poststop": structs.TaskState{State: "pending", Restarts: 0},
"main": {State: "running", Restarts: 1},
"prestart-oneshot": {State: "dead", Restarts: 0},
"prestart-sidecar": {State: "running", Restarts: 0},
"poststart-oneshot": {State: "dead", Restarts: 0},
"poststart-sidecar": {State: "running", Restarts: 0},
"poststop": {State: "pending", Restarts: 0},
},
},
{
@ -668,12 +668,12 @@ func TestAllocRunner_Lifecycle_Restart(t *testing.T) {
return nil
},
expectedAfter: map[string]structs.TaskState{
"main": structs.TaskState{State: "dead", Restarts: 1},
"prestart-oneshot": structs.TaskState{State: "dead", Restarts: 0},
"prestart-sidecar": structs.TaskState{State: "dead", Restarts: 0},
"poststart-oneshot": structs.TaskState{State: "dead", Restarts: 0},
"poststart-sidecar": structs.TaskState{State: "dead", Restarts: 0},
"poststop": structs.TaskState{State: "dead", Restarts: 0},
"main": {State: "dead", Restarts: 1},
"prestart-oneshot": {State: "dead", Restarts: 0},
"prestart-sidecar": {State: "dead", Restarts: 0},
"poststart-oneshot": {State: "dead", Restarts: 0},
"poststart-sidecar": {State: "dead", Restarts: 0},
"poststop": {State: "dead", Restarts: 0},
},
},
{
@ -692,12 +692,12 @@ func TestAllocRunner_Lifecycle_Restart(t *testing.T) {
return nil
},
expectedAfter: map[string]structs.TaskState{
"main": structs.TaskState{State: "dead", Restarts: 1},
"prestart-oneshot": structs.TaskState{State: "dead", Restarts: 0},
"prestart-sidecar": structs.TaskState{State: "dead", Restarts: 0},
"poststart-oneshot": structs.TaskState{State: "dead", Restarts: 0},
"poststart-sidecar": structs.TaskState{State: "dead", Restarts: 0},
"poststop": structs.TaskState{State: "dead", Restarts: 0},
"main": {State: "dead", Restarts: 1},
"prestart-oneshot": {State: "dead", Restarts: 0},
"prestart-sidecar": {State: "dead", Restarts: 0},
"poststart-oneshot": {State: "dead", Restarts: 0},
"poststart-sidecar": {State: "dead", Restarts: 0},
"poststop": {State: "dead", Restarts: 0},
},
},
{
@ -715,12 +715,12 @@ func TestAllocRunner_Lifecycle_Restart(t *testing.T) {
return nil
},
expectedAfter: map[string]structs.TaskState{
"main": structs.TaskState{State: "dead", Restarts: 1},
"prestart-oneshot": structs.TaskState{State: "dead", Restarts: 0},
"prestart-sidecar": structs.TaskState{State: "dead", Restarts: 0},
"poststart-oneshot": structs.TaskState{State: "dead", Restarts: 0},
"poststart-sidecar": structs.TaskState{State: "dead", Restarts: 0},
"poststop": structs.TaskState{State: "dead", Restarts: 0},
"main": {State: "dead", Restarts: 1},
"prestart-oneshot": {State: "dead", Restarts: 0},
"prestart-sidecar": {State: "dead", Restarts: 0},
"poststart-oneshot": {State: "dead", Restarts: 0},
"poststart-sidecar": {State: "dead", Restarts: 0},
"poststop": {State: "dead", Restarts: 0},
},
},
{
@ -738,12 +738,12 @@ func TestAllocRunner_Lifecycle_Restart(t *testing.T) {
return nil
},
expectedAfter: map[string]structs.TaskState{
"main": structs.TaskState{State: "dead", Restarts: 1},
"prestart-oneshot": structs.TaskState{State: "dead", Restarts: 0},
"prestart-sidecar": structs.TaskState{State: "dead", Restarts: 0},
"poststart-oneshot": structs.TaskState{State: "dead", Restarts: 0},
"poststart-sidecar": structs.TaskState{State: "dead", Restarts: 0},
"poststop": structs.TaskState{State: "dead", Restarts: 0},
"main": {State: "dead", Restarts: 1},
"prestart-oneshot": {State: "dead", Restarts: 0},
"prestart-sidecar": {State: "dead", Restarts: 0},
"poststart-oneshot": {State: "dead", Restarts: 0},
"poststart-sidecar": {State: "dead", Restarts: 0},
"poststop": {State: "dead", Restarts: 0},
},
},
{
@ -764,12 +764,12 @@ func TestAllocRunner_Lifecycle_Restart(t *testing.T) {
},
expectedErr: "Task not running",
expectedAfter: map[string]structs.TaskState{
"main": structs.TaskState{State: "dead", Restarts: 1},
"prestart-oneshot": structs.TaskState{State: "dead", Restarts: 0},
"prestart-sidecar": structs.TaskState{State: "dead", Restarts: 0},
"poststart-oneshot": structs.TaskState{State: "dead", Restarts: 0},
"poststart-sidecar": structs.TaskState{State: "dead", Restarts: 0},
"poststop": structs.TaskState{State: "dead", Restarts: 0},
"main": {State: "dead", Restarts: 1},
"prestart-oneshot": {State: "dead", Restarts: 0},
"prestart-sidecar": {State: "dead", Restarts: 0},
"poststart-oneshot": {State: "dead", Restarts: 0},
"poststart-sidecar": {State: "dead", Restarts: 0},
"poststop": {State: "dead", Restarts: 0},
},
},
{
@ -778,12 +778,12 @@ func TestAllocRunner_Lifecycle_Restart(t *testing.T) {
return ar.RestartTask("prestart-sidecar", ev)
},
expectedAfter: map[string]structs.TaskState{
"main": structs.TaskState{State: "running", Restarts: 0},
"prestart-oneshot": structs.TaskState{State: "dead", Restarts: 0},
"prestart-sidecar": structs.TaskState{State: "running", Restarts: 1},
"poststart-oneshot": structs.TaskState{State: "dead", Restarts: 0},
"poststart-sidecar": structs.TaskState{State: "running", Restarts: 0},
"poststop": structs.TaskState{State: "pending", Restarts: 0},
"main": {State: "running", Restarts: 0},
"prestart-oneshot": {State: "dead", Restarts: 0},
"prestart-sidecar": {State: "running", Restarts: 1},
"poststart-oneshot": {State: "dead", Restarts: 0},
"poststart-sidecar": {State: "running", Restarts: 0},
"poststop": {State: "pending", Restarts: 0},
},
},
{
@ -792,12 +792,12 @@ func TestAllocRunner_Lifecycle_Restart(t *testing.T) {
return ar.RestartTask("poststart-sidecar", ev)
},
expectedAfter: map[string]structs.TaskState{
"main": structs.TaskState{State: "running", Restarts: 0},
"prestart-oneshot": structs.TaskState{State: "dead", Restarts: 0},
"prestart-sidecar": structs.TaskState{State: "running", Restarts: 0},
"poststart-oneshot": structs.TaskState{State: "dead", Restarts: 0},
"poststart-sidecar": structs.TaskState{State: "running", Restarts: 1},
"poststop": structs.TaskState{State: "pending", Restarts: 0},
"main": {State: "running", Restarts: 0},
"prestart-oneshot": {State: "dead", Restarts: 0},
"prestart-sidecar": {State: "running", Restarts: 0},
"poststart-oneshot": {State: "dead", Restarts: 0},
"poststart-sidecar": {State: "running", Restarts: 1},
"poststop": {State: "pending", Restarts: 0},
},
},
}

2
client/allocrunner/network_manager_linux.go
View File

@ -184,7 +184,7 @@ func newNetworkConfigurator(log hclog.Logger, alloc *structs.Allocation, config
switch {
case netMode == "bridge":
c, err := newBridgeNetworkConfigurator(log, config.BridgeNetworkName, config.BridgeNetworkAllocSubnet, config.CNIPath, ignorePortMappingHostIP)
c, err := newBridgeNetworkConfigurator(log, config.BridgeNetworkName, config.BridgeNetworkAllocSubnet, config.BridgeNetworkHairpinMode, config.CNIPath, ignorePortMappingHostIP)
if err != nil {
return nil, err
}

24
client/allocrunner/networking_bridge_linux.go
View File

@ -3,6 +3,7 @@ package allocrunner
import (
"context"
"fmt"
"text/template"
"github.com/coreos/go-iptables/iptables"
hclog "github.com/hashicorp/go-hclog"
@ -28,6 +29,8 @@ const (
cniAdminChainName = "NOMAD-ADMIN"
)
var nomadBridgeTmpl = template.Must(template.New("cniConf").Parse(nomadCNIConfigTemplate))
// bridgeNetworkConfigurator is a NetworkConfigurator which adds the alloc to a
// shared bridge, configures masquerading for egress traffic and port mapping
// for ingress
@ -35,14 +38,16 @@ type bridgeNetworkConfigurator struct {
cni *cniNetworkConfigurator
allocSubnet string
bridgeName string
hairpinMode bool
logger hclog.Logger
}
func newBridgeNetworkConfigurator(log hclog.Logger, bridgeName, ipRange, cniPath string, ignorePortMappingHostIP bool) (*bridgeNetworkConfigurator, error) {
func newBridgeNetworkConfigurator(log hclog.Logger, bridgeName, ipRange string, hairpinMode bool, cniPath string, ignorePortMappingHostIP bool) (*bridgeNetworkConfigurator, error) {
b := &bridgeNetworkConfigurator{
bridgeName: bridgeName,
allocSubnet: ipRange,
hairpinMode: hairpinMode,
logger: log,
}
@ -54,7 +59,7 @@ func newBridgeNetworkConfigurator(log hclog.Logger, bridgeName, ipRange, cniPath
b.allocSubnet = defaultNomadAllocSubnet
}
c, err := newCNINetworkConfiguratorWithConf(log, cniPath, bridgeNetworkAllocIfPrefix, ignorePortMappingHostIP, buildNomadBridgeNetConfig(b.bridgeName, b.allocSubnet))
c, err := newCNINetworkConfiguratorWithConf(log, cniPath, bridgeNetworkAllocIfPrefix, ignorePortMappingHostIP, buildNomadBridgeNetConfig(*b))
if err != nil {
return nil, err
}
@ -134,8 +139,12 @@ func (b *bridgeNetworkConfigurator) Teardown(ctx context.Context, alloc *structs
return b.cni.Teardown(ctx, alloc, spec)
}
func buildNomadBridgeNetConfig(bridgeName, subnet string) []byte {
return []byte(fmt.Sprintf(nomadCNIConfigTemplate, bridgeName, subnet, cniAdminChainName))
func buildNomadBridgeNetConfig(b bridgeNetworkConfigurator) []byte {
return []byte(fmt.Sprintf(nomadCNIConfigTemplate,
b.bridgeName,
b.hairpinMode,
b.allocSubnet,
cniAdminChainName))
}
const nomadCNIConfigTemplate = `{
@ -147,16 +156,17 @@ const nomadCNIConfigTemplate = `{
},
{
"type": "bridge",
"bridge": "%s",
"bridge": %q,
"ipMasq": true,
"isGateway": true,
"forceAddress": true,
"hairpinMode": %v,
"ipam": {
"type": "host-local",
"ranges": [
[
{
"subnet": "%s"
"subnet": %q
}
]
],
@ -168,7 +178,7 @@ const nomadCNIConfigTemplate = `{
{
"type": "firewall",
"backend": "iptables",
"iptablesAdminChainName": "%s"
"iptablesAdminChainName": %q
},
{
"type": "portmap",

48
client/allocrunner/networking_bridge_linux_test.go Normal file
View File

@ -0,0 +1,48 @@
package allocrunner
import (
"encoding/json"
"testing"
"github.com/hashicorp/nomad/ci"
"github.com/shoenig/test/must"
)
func Test_buildNomadBridgeNetConfig(t *testing.T) {
ci.Parallel(t)
testCases := []struct {
name string
b *bridgeNetworkConfigurator
}{
{
name: "empty",
b: &bridgeNetworkConfigurator{},
},
{
name: "hairpin",
b: &bridgeNetworkConfigurator{
bridgeName: defaultNomadBridgeName,
allocSubnet: defaultNomadAllocSubnet,
hairpinMode: true,
},
},
{
name: "bad_input",
b: &bridgeNetworkConfigurator{
bridgeName: `bad"`,
allocSubnet: defaultNomadAllocSubnet,
hairpinMode: true,
},
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
tc := tc
ci.Parallel(t)
bCfg := buildNomadBridgeNetConfig(*tc.b)
// Validate that the JSON created is rational
must.True(t, json.Valid(bCfg))
})
}
}

4
client/config/config.go
View File

@ -261,6 +261,10 @@ type Config struct {
// networking mode. This defaults to 'nomad' if not set
BridgeNetworkName string
// BridgeNetworkHairpinMode is whether or not to enable hairpin mode on the
// internal bridge network
BridgeNetworkHairpinMode bool
// BridgeNetworkAllocSubnet is the IP subnet to use for address allocation
// for allocations in bridge networking mode. Subnet must be in CIDR
// notation

1
client/fingerprint/bridge_default.go
View File

@ -1,5 +1,4 @@
//go:build !linux
// +build !linux
package fingerprint

6
client/fingerprint/bridge_linux.go
View File

@ -1,3 +1,5 @@
//go:build linux
package fingerprint
import (
@ -5,6 +7,7 @@ import (
"fmt"
"os"
"regexp"
"strconv"
"github.com/hashicorp/go-multierror"
"github.com/hashicorp/nomad/nomad/structs"
@ -35,6 +38,9 @@ func (f *BridgeFingerprint) Fingerprint(req *FingerprintRequest, resp *Fingerpri
}},
}
resp.AddAttribute("nomad.bridge.hairpin_mode",
strconv.FormatBool(req.Config.BridgeNetworkHairpinMode))
resp.Detected = true
return nil
}

1
command/agent/agent.go
View File

@ -788,6 +788,7 @@ func convertClientConfig(agentConfig *Config) (*clientconfig.Config, error) {
conf.CNIConfigDir = agentConfig.Client.CNIConfigDir
conf.BridgeNetworkName = agentConfig.Client.BridgeNetworkName
conf.BridgeNetworkAllocSubnet = agentConfig.Client.BridgeNetworkSubnet
conf.BridgeNetworkHairpinMode = agentConfig.Client.BridgeNetworkHairpinMode
for _, hn := range agentConfig.Client.HostNetworks {
conf.HostNetworks[hn.Name] = hn

8
command/agent/config.go
View File

@ -315,6 +315,10 @@ type ClientConfig struct {
// the host
BridgeNetworkSubnet string `hcl:"bridge_network_subnet"`
// BridgeNetworkHairpinMode is whether or not to enable hairpin mode on the
// internal bridge network
BridgeNetworkHairpinMode bool `hcl:"bridge_network_hairpin_mode"`
// HostNetworks describes the different host networks available to the host
// if the host uses multiple interfaces
HostNetworks []*structs.ClientHostNetworkConfig `hcl:"host_network"`
@ -2137,6 +2141,10 @@ func (a *ClientConfig) Merge(b *ClientConfig) *ClientConfig {
result.BridgeNetworkSubnet = b.BridgeNetworkSubnet
}
if b.BridgeNetworkHairpinMode {
result.BridgeNetworkHairpinMode = true
}
result.HostNetworks = a.HostNetworks
if len(b.HostNetworks) != 0 {

7
website/content/docs/configuration/client.mdx
View File

@ -147,12 +147,17 @@ client {
configuration.
- `bridge_network_name` `(string: "nomad")` - Sets the name of the bridge to be
created by nomad for allocations running with bridge networking mode on the
created by Nomad for allocations running with bridge networking mode on the
client.
- `bridge_network_subnet` `(string: "172.26.64.0/20")` - Specifies the subnet
which the client will use to allocate IP addresses from.
- `bridge_network_hairpin_mode` `(bool: false)` - Specifies if hairpin mode
is enabled on the network bridge created by Nomad for allocations running
with bridge networking mode on this client. You may use the corresponding
node attribute `nomad.bridge.hairpin_mode` in constraints.
- `artifact` <code>([Artifact](#artifact-parameters): varied)</code> -
Specifies controls on the behavior of task
[`artifact`](/nomad/docs/job-specification/artifact) blocks.