From 4fcb7394e95f8471ac0d3a561b65738c0c8bb06d Mon Sep 17 00:00:00 2001 From: Danielle Lancashire Date: Sat, 31 Aug 2019 16:36:47 +0200 Subject: [PATCH 1/4] client: Fix memory fingerprinting on 32bit Also introduce regression ci for 32 bit fingerprinting --- .circleci/config.yml | 9 +++++++++ client/fingerprint/memory.go | 15 +++++---------- client/fingerprint/memory_test.go | 24 ++++++------------------ 3 files changed, 20 insertions(+), 28 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 92d95411b..6e830cd3e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -54,6 +54,11 @@ workflows: name: "test-shared-exec" test_packages: "./drivers/shared/executor" <<: *IGNORE_FOR_UI_BRANCHES + - test-container: + name: "test-32bit-fingerprinting" + test_packages: "./client/fingerprint" + goarch: "386" + <<: *IGNORE_FOR_UI_BRANCHES - test-rkt: <<: *IGNORE_FOR_UI_BRANCHES - test-e2e: @@ -118,11 +123,15 @@ jobs: exclude_packages: type: string default: "" + goarch: + type: string + default: "amd64" environment: <<: *COMMON_ENVS GOTEST_PKGS: "<< parameters.test_packages >>" GOTEST_PKGS_EXCLUDE: "<< parameters.exclude_packages >>" GOPATH: /go + GOARCH: "<< parameters.goarch >>" steps: - checkout - run: make deps diff --git a/client/fingerprint/memory.go b/client/fingerprint/memory.go index ad28ed2ec..04436e9b0 100644 --- a/client/fingerprint/memory.go +++ b/client/fingerprint/memory.go @@ -8,7 +8,7 @@ import ( "github.com/shirou/gopsutil/mem" ) -const bytesInMB = 1024 * 1024 +const bytesInMB int64 = 1024 * 1024 // MemoryFingerprint is used to fingerprint the available memory on the node type MemoryFingerprint struct { @@ -25,10 +25,10 @@ func NewMemoryFingerprint(logger log.Logger) Fingerprint { } func (f *MemoryFingerprint) Fingerprint(req *FingerprintRequest, resp *FingerprintResponse) error { - var totalMemory int + var totalMemory int64 cfg := req.Config if cfg.MemoryMB != 0 { - totalMemory = cfg.MemoryMB * bytesInMB + totalMemory = int64(cfg.MemoryMB) * bytesInMB } else { memInfo, err := mem.VirtualMemory() if err != nil { @@ -36,21 +36,16 @@ func (f *MemoryFingerprint) Fingerprint(req *FingerprintRequest, resp *Fingerpri return err } if memInfo.Total > 0 { - totalMemory = int(memInfo.Total) + totalMemory = int64(memInfo.Total) } } if totalMemory > 0 { resp.AddAttribute("memory.totalbytes", fmt.Sprintf("%d", totalMemory)) - // COMPAT(0.10): Remove in 0.10 - resp.Resources = &structs.Resources{ - MemoryMB: totalMemory / bytesInMB, - } - resp.NodeResources = &structs.NodeResources{ Memory: structs.NodeMemoryResources{ - MemoryMB: int64(totalMemory / bytesInMB), + MemoryMB: totalMemory / bytesInMB, }, } } diff --git a/client/fingerprint/memory_test.go b/client/fingerprint/memory_test.go index 6624b9225..c04b0f947 100644 --- a/client/fingerprint/memory_test.go +++ b/client/fingerprint/memory_test.go @@ -11,6 +11,8 @@ import ( ) func TestMemoryFingerprint(t *testing.T) { + require := require.New(t) + f := NewMemoryFingerprint(testlog.HCLogger(t)) node := &structs.Node{ Attributes: make(map[string]string), @@ -19,24 +21,11 @@ func TestMemoryFingerprint(t *testing.T) { request := &FingerprintRequest{Config: &config.Config{}, Node: node} var response FingerprintResponse err := f.Fingerprint(request, &response) - if err != nil { - t.Fatalf("err: %v", err) - } + require.NoError(err) assertNodeAttributeContains(t, response.Attributes, "memory.totalbytes") - - if response.Resources == nil { - t.Fatalf("response resources should not be nil") - } - - // COMPAT(0.10): Remove in 0.10 - if response.Resources.MemoryMB == 0 { - t.Fatalf("Expected node.Resources.MemoryMB to be non-zero") - } - - if response.NodeResources.Memory.MemoryMB == 0 { - t.Fatalf("Expected node.Resources.MemoryMB to be non-zero") - } + require.NotNil(response.NodeResources, "expected response NodeResources to not be nil") + require.NotZero(response.NodeResources.Memory.MemoryMB, "expected memory to be non-zero") } func TestMemoryFingerprint_Override(t *testing.T) { @@ -55,7 +44,6 @@ func TestMemoryFingerprint_Override(t *testing.T) { assertNodeAttributeContains(t, response.Attributes, "memory.totalbytes") require := require.New(t) - require.NotNil(response.Resources) - require.Equal(response.Resources.MemoryMB, memoryMB) + require.NotNil(response.NodeResources) require.EqualValues(response.NodeResources.Memory.MemoryMB, memoryMB) } From d970d5ee00198b2c36fd932214fe83a5a97944b1 Mon Sep 17 00:00:00 2001 From: Danielle Lancashire Date: Sat, 31 Aug 2019 18:34:22 +0200 Subject: [PATCH 2/4] ci: Support non amd64 architectures --- .circleci/config.yml | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6e830cd3e..3e0989cb7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -54,8 +54,10 @@ workflows: name: "test-shared-exec" test_packages: "./drivers/shared/executor" <<: *IGNORE_FOR_UI_BRANCHES - - test-container: - name: "test-32bit-fingerprinting" + - test-machine: + name: "test-32bit" + # Currently we only explicitly test fingerprinting on 32bit + # architectures. test_packages: "./client/fingerprint" goarch: "386" <<: *IGNORE_FOR_UI_BRANCHES @@ -131,7 +133,7 @@ jobs: GOTEST_PKGS: "<< parameters.test_packages >>" GOTEST_PKGS_EXCLUDE: "<< parameters.exclude_packages >>" GOPATH: /go - GOARCH: "<< parameters.goarch >>" + GOTESTARCH: "<< parameters.goarch >>" steps: - checkout - run: make deps @@ -200,17 +202,30 @@ jobs: executor: type: string default: "go-machine-recent" + goarch: + type: string + default: "amd64" environment: <<: *COMMON_ENVS GOTEST_PKGS_EXCLUDE: "<< parameters.exclude_packages >>" GOTEST_PKGS: "<< parameters.test_packages >>" GOPATH: /home/circleci/go + GOTESTARCH: "<< parameters.goarch >>" steps: - checkout - install-golang - install-protoc - install-consul - install-vault + - run: + name: Install 32bit gcc libs + command: | + if [ ! -z $GOTESTARCH ] && [ $GOTESTARCH == "386" ]; then + sudo apt-get update + sudo apt-get install -y gcc-multilib + else + echo "Skipping 32bit lib installation while building for not 386" + fi - run: PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make bootstrap - run-tests - store_test_results: @@ -339,6 +354,10 @@ commands: unset GOTEST_PKGS fi + if [ ! -z $GOTESTARCH ]; then + export GOARCH="$GOTESTARCH"; + fi + mkdir -p /tmp/test-reports sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make generate-structs sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make test-nomad From 67715d846e214ce77c9196c36ca2beb3266f4b84 Mon Sep 17 00:00:00 2001 From: Danielle Lancashire Date: Wed, 4 Sep 2019 17:09:30 +0200 Subject: [PATCH 3/4] fingerprint: Restore support for legacy memory fingerprint --- client/fingerprint/memory.go | 8 +++++++- client/fingerprint/memory_test.go | 4 ++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/client/fingerprint/memory.go b/client/fingerprint/memory.go index 04436e9b0..8320f9bb7 100644 --- a/client/fingerprint/memory.go +++ b/client/fingerprint/memory.go @@ -43,9 +43,15 @@ func (f *MemoryFingerprint) Fingerprint(req *FingerprintRequest, resp *Fingerpri if totalMemory > 0 { resp.AddAttribute("memory.totalbytes", fmt.Sprintf("%d", totalMemory)) + memoryMB := totalMemory / bytesInMB + + resp.Resources = &structs.Resources{ + MemoryMB: int(memoryMB), + } + resp.NodeResources = &structs.NodeResources{ Memory: structs.NodeMemoryResources{ - MemoryMB: totalMemory / bytesInMB, + MemoryMB: memoryMB, }, } } diff --git a/client/fingerprint/memory_test.go b/client/fingerprint/memory_test.go index c04b0f947..8635b55fd 100644 --- a/client/fingerprint/memory_test.go +++ b/client/fingerprint/memory_test.go @@ -24,6 +24,8 @@ func TestMemoryFingerprint(t *testing.T) { require.NoError(err) assertNodeAttributeContains(t, response.Attributes, "memory.totalbytes") + require.NotNil(response.Resources, "expected response Resources to not be nil") + require.NotZero(response.Resources.MemoryMB, "expected memory to be non-zero") require.NotNil(response.NodeResources, "expected response NodeResources to not be nil") require.NotZero(response.NodeResources.Memory.MemoryMB, "expected memory to be non-zero") } @@ -44,6 +46,8 @@ func TestMemoryFingerprint_Override(t *testing.T) { assertNodeAttributeContains(t, response.Attributes, "memory.totalbytes") require := require.New(t) + require.NotNil(response.Resources) + require.EqualValues(response.Resources.MemoryMB, memoryMB) require.NotNil(response.NodeResources) require.EqualValues(response.NodeResources.Memory.MemoryMB, memoryMB) } From aa5605fce1ab442b61a5a08c8a677eeb72ec44ff Mon Sep 17 00:00:00 2001 From: Danielle Date: Wed, 4 Sep 2019 17:38:35 +0200 Subject: [PATCH 4/4] fingerprint: Add backwards compatibility comment Co-Authored-By: Michael Schurter --- client/fingerprint/memory.go | 1 + 1 file changed, 1 insertion(+) diff --git a/client/fingerprint/memory.go b/client/fingerprint/memory.go index 8320f9bb7..df5e24c33 100644 --- a/client/fingerprint/memory.go +++ b/client/fingerprint/memory.go @@ -45,6 +45,7 @@ func (f *MemoryFingerprint) Fingerprint(req *FingerprintRequest, resp *Fingerpri memoryMB := totalMemory / bytesInMB + // COMPAT(0.10): Unused since 0.9. resp.Resources = &structs.Resources{ MemoryMB: int(memoryMB), }