luxolus/open-nomad
2
0
Fork 0
Code Issues 1 Pull requests Packages Releases Wiki Activity

e2e: give containers access to dnsmasq DNS (#8536)

Browse source 
By default, Docker containers get /etc/resolv.conf bound into the container
with the localhost entry stripped out. In order to resolve using the host's
dnsmasq, we need to make sure the container uses the docker0 IP as its
nameserver and that dnsmasq is listening on that port and forwarding to either
the AWS VPC DNS (so that we can query private resources like EFS) or to the
Consul DNS.
This commit is contained in:
Tim Gross 2020-07-24 14:09:18 -04:00 committed by GitHub
parent 2d0b80a0ed
commit d0b03cad7c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 44 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
e2e/terraform/packer/linux/setup.sh
View file

@ -115,7 +115,7 @@ echo "Install Podman"
sudo sh -c "echo 'deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_${VERSION_ID}/ /' > /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list"
curl -L https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_${VERSION_ID}/Release.key | sudo apt-key add -
sudo apt-get update -qq
sudo apt-get -qq -y install podman
sudo apt-get -qq -y install podman
# get catatonit (to check podman --init switch)
cd /tmp
@ -132,21 +132,26 @@ wget -P /tmp https://releases.hashicorp.com/nomad-driver-podman/${latest_podman}
sudo unzip /tmp/nomad-driver-podman_${latest_podman}_linux_amd64.zip -d $NOMADPLUGINDIR
sudo chmod +x $NOMADPLUGINDIR/nomad-driver-podman
# disable systemd-resolved and configure dnsmasq
# to forward local requests to consul
# disable systemd-resolved and configure dnsmasq to forward local requests to
# consul. the resolver files need to dynamic configuration based on the VPC
# address and docker bridge IP, so those will be rewritten at boot time.
sudo systemctl disable systemd-resolved.service
sudo rm /etc/resolv.conf
echo "nameserver 8.8.8.8" | sudo tee /etc/resolv.conf
echo '
port=53
resolv-file=/var/run/dnsmasq/resolv.conf
bind-interfaces
interface=docker0
interface=lo
interface=eth0
listen-address=127.0.0.1
server=/consul/127.0.0.1#8600
' | sudo tee /etc/dnsmasq.d/default
# add our hostname to etc/hosts
echo "127.0.0.1 $(hostname)" | sudo tee -a /etc/hosts
# this is going to be overwritten at provisioning time, but we need something
# here or we can't fetch binaries to do the provisioning
echo 'nameserver 8.8.8.8' > /tmp/resolv.conf
sudo mv /tmp/resolv.conf /etc/resolv.conf
sudo systemctl restart dnsmasq
# enable cgroup_memory and swap

21
e2e/terraform/shared/config/provision-client.sh
View file

@ -22,11 +22,24 @@ sleep 10
# Add hostname to /etc/hosts
echo "127.0.0.1 $(hostname)" | sudo tee --append /etc/hosts
# Add Docker bridge network IP to /etc/resolv.conf (at the top)
# Use dnsmasq first and then docker bridge network for DNS resolution
DOCKER_BRIDGE_IP_ADDRESS=$(/usr/local/bin/sockaddr eval 'GetInterfaceIP "docker0"')
echo "nameserver $DOCKER_BRIDGE_IP_ADDRESS" | sudo tee /etc/resolv.conf.new
cat /etc/resolv.conf | sudo tee --append /etc/resolv.conf.new
sudo mv /etc/resolv.conf.new /etc/resolv.conf
cat <<EOF > /tmp/resolv.conf
nameserver 127.0.0.1
nameserver $DOCKER_BRIDGE_IP_ADDRESS
EOF
sudo mv /tmp/resolv.conf /etc/resolv.conf
# need to get the AWS DNS address from the VPC...
# this is pretty hacky but will work for any typical case
MAC=$(curl -s --fail http://169.254.169.254/latest/meta-data/mac)
CIDR_BLOCK=$(curl -s --fail "http://169.254.169.254/latest/meta-data/network/interfaces/macs/$MAC/vpc-ipv4-cidr-block")
VPC_DNS_ROOT=$(echo "$CIDR_BLOCK" | cut -d'.' -f1-3)
echo "nameserver ${VPC_DNS_ROOT}.2" > /tmp/dnsmasq-resolv.conf
sudo mv /tmp/dnsmasq-resolv.conf /var/run/dnsmasq/resolv.conf
sudo systemctl restart dnsmasq
sudo systemctl restart docker
# Nomad

20
e2e/terraform/shared/config/provision-server.sh
View file

@ -36,11 +36,21 @@ sudo systemctl restart vault.service
# Add hostname to /etc/hosts
echo "127.0.0.1 $(hostname)" | sudo tee --append /etc/hosts
# Add Docker bridge network IP to /etc/resolv.conf (at the top)
DOCKER_BRIDGE_IP_ADDRESS=$(/usr/local/bin/sockaddr eval 'GetInterfaceIP "docker0"')
echo "nameserver $DOCKER_BRIDGE_IP_ADDRESS" | sudo tee /etc/resolv.conf.new
cat /etc/resolv.conf | sudo tee --append /etc/resolv.conf.new
sudo mv /etc/resolv.conf.new /etc/resolv.conf
# Use dnsmasq for DNS resolution
echo "nameserver 127.0.0.1" > /tmp/resolv.conf
sudo mv /tmp/resolv.conf /etc/resolv.conf
# need to get the AWS DNS address from the VPC...
# this is pretty hacky but will work for any typical case
MAC=$(curl -s --fail http://169.254.169.254/latest/meta-data/mac)
CIDR_BLOCK=$(curl -s --fail "http://169.254.169.254/latest/meta-data/network/interfaces/macs/$MAC/vpc-ipv4-cidr-block")
VPC_DNS_ROOT=$(echo "$CIDR_BLOCK" | cut -d'.' -f1-3)
{
echo "nameserver ${VPC_DNS_ROOT}.2"
} > /tmp/dnsmasq-resolv.conf
sudo mv /tmp/dnsmasq-resolv.conf /var/run/dnsmasq/resolv.conf
sudo systemctl restart dnsmasq
# Nomad