diff --git a/e2e/terraform/packer/linux/setup.sh b/e2e/terraform/packer/linux/setup.sh index c805254c7..444ae5341 100755 --- a/e2e/terraform/packer/linux/setup.sh +++ b/e2e/terraform/packer/linux/setup.sh @@ -115,7 +115,7 @@ echo "Install Podman" sudo sh -c "echo 'deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_${VERSION_ID}/ /' > /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list" curl -L https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_${VERSION_ID}/Release.key | sudo apt-key add - sudo apt-get update -qq -sudo apt-get -qq -y install podman +sudo apt-get -qq -y install podman # get catatonit (to check podman --init switch) cd /tmp @@ -132,21 +132,26 @@ wget -P /tmp https://releases.hashicorp.com/nomad-driver-podman/${latest_podman} sudo unzip /tmp/nomad-driver-podman_${latest_podman}_linux_amd64.zip -d $NOMADPLUGINDIR sudo chmod +x $NOMADPLUGINDIR/nomad-driver-podman -# disable systemd-resolved and configure dnsmasq -# to forward local requests to consul +# disable systemd-resolved and configure dnsmasq to forward local requests to +# consul. the resolver files need to dynamic configuration based on the VPC +# address and docker bridge IP, so those will be rewritten at boot time. sudo systemctl disable systemd-resolved.service -sudo rm /etc/resolv.conf -echo "nameserver 8.8.8.8" | sudo tee /etc/resolv.conf echo ' port=53 resolv-file=/var/run/dnsmasq/resolv.conf bind-interfaces +interface=docker0 +interface=lo +interface=eth0 listen-address=127.0.0.1 server=/consul/127.0.0.1#8600 ' | sudo tee /etc/dnsmasq.d/default -# add our hostname to etc/hosts -echo "127.0.0.1 $(hostname)" | sudo tee -a /etc/hosts +# this is going to be overwritten at provisioning time, but we need something +# here or we can't fetch binaries to do the provisioning +echo 'nameserver 8.8.8.8' > /tmp/resolv.conf +sudo mv /tmp/resolv.conf /etc/resolv.conf + sudo systemctl restart dnsmasq # enable cgroup_memory and swap diff --git a/e2e/terraform/shared/config/provision-client.sh b/e2e/terraform/shared/config/provision-client.sh index 011eaa304..f431c7b59 100755 --- a/e2e/terraform/shared/config/provision-client.sh +++ b/e2e/terraform/shared/config/provision-client.sh @@ -22,11 +22,24 @@ sleep 10 # Add hostname to /etc/hosts echo "127.0.0.1 $(hostname)" | sudo tee --append /etc/hosts -# Add Docker bridge network IP to /etc/resolv.conf (at the top) +# Use dnsmasq first and then docker bridge network for DNS resolution DOCKER_BRIDGE_IP_ADDRESS=$(/usr/local/bin/sockaddr eval 'GetInterfaceIP "docker0"') -echo "nameserver $DOCKER_BRIDGE_IP_ADDRESS" | sudo tee /etc/resolv.conf.new -cat /etc/resolv.conf | sudo tee --append /etc/resolv.conf.new -sudo mv /etc/resolv.conf.new /etc/resolv.conf +cat < /tmp/resolv.conf +nameserver 127.0.0.1 +nameserver $DOCKER_BRIDGE_IP_ADDRESS +EOF +sudo mv /tmp/resolv.conf /etc/resolv.conf + +# need to get the AWS DNS address from the VPC... +# this is pretty hacky but will work for any typical case +MAC=$(curl -s --fail http://169.254.169.254/latest/meta-data/mac) +CIDR_BLOCK=$(curl -s --fail "http://169.254.169.254/latest/meta-data/network/interfaces/macs/$MAC/vpc-ipv4-cidr-block") +VPC_DNS_ROOT=$(echo "$CIDR_BLOCK" | cut -d'.' -f1-3) +echo "nameserver ${VPC_DNS_ROOT}.2" > /tmp/dnsmasq-resolv.conf +sudo mv /tmp/dnsmasq-resolv.conf /var/run/dnsmasq/resolv.conf + +sudo systemctl restart dnsmasq +sudo systemctl restart docker # Nomad diff --git a/e2e/terraform/shared/config/provision-server.sh b/e2e/terraform/shared/config/provision-server.sh index 3c786d085..2c5736176 100755 --- a/e2e/terraform/shared/config/provision-server.sh +++ b/e2e/terraform/shared/config/provision-server.sh @@ -36,11 +36,21 @@ sudo systemctl restart vault.service # Add hostname to /etc/hosts echo "127.0.0.1 $(hostname)" | sudo tee --append /etc/hosts -# Add Docker bridge network IP to /etc/resolv.conf (at the top) -DOCKER_BRIDGE_IP_ADDRESS=$(/usr/local/bin/sockaddr eval 'GetInterfaceIP "docker0"') -echo "nameserver $DOCKER_BRIDGE_IP_ADDRESS" | sudo tee /etc/resolv.conf.new -cat /etc/resolv.conf | sudo tee --append /etc/resolv.conf.new -sudo mv /etc/resolv.conf.new /etc/resolv.conf +# Use dnsmasq for DNS resolution +echo "nameserver 127.0.0.1" > /tmp/resolv.conf +sudo mv /tmp/resolv.conf /etc/resolv.conf + +# need to get the AWS DNS address from the VPC... +# this is pretty hacky but will work for any typical case +MAC=$(curl -s --fail http://169.254.169.254/latest/meta-data/mac) +CIDR_BLOCK=$(curl -s --fail "http://169.254.169.254/latest/meta-data/network/interfaces/macs/$MAC/vpc-ipv4-cidr-block") +VPC_DNS_ROOT=$(echo "$CIDR_BLOCK" | cut -d'.' -f1-3) +{ + echo "nameserver ${VPC_DNS_ROOT}.2" +} > /tmp/dnsmasq-resolv.conf +sudo mv /tmp/dnsmasq-resolv.conf /var/run/dnsmasq/resolv.conf + +sudo systemctl restart dnsmasq # Nomad