From c9468eaabd9043561a5305df38ca35a37857df33 Mon Sep 17 00:00:00 2001 From: diginc Date: Thu, 20 Oct 2016 22:36:55 -0500 Subject: [PATCH 1/4] Pi-hole and Web versions update + new stuff * New code in shell scripts required killing systemctl in images * Added tests for the quick IPv6 env hack I made a while back * Stopped requiring the need for update.sh to copy gravity for alpine * (moved modifications to dockerfile) * One step closer to unified start script for both OS versions * Optimzed some test code --- AdminLTE | 2 +- AdminLTE_version.txt | 2 +- alpine.docker | 9 +- alpine/gravity.sh | 359 ------------------------------ alpine/start.sh | 10 +- common_start.sh | 49 ++-- debian-armhf.docker | 8 + debian.docker | 8 +- debian/start.sh | 1 + pi-hole | 2 +- pi-hole_version.txt | 2 +- test/conftest.py | 9 +- test/test_000_build_containers.py | 2 +- test/test_start.py | 58 +++-- update.sh | 4 - 15 files changed, 103 insertions(+), 422 deletions(-) delete mode 100755 alpine/gravity.sh diff --git a/AdminLTE b/AdminLTE index f112893..b270c61 160000 --- a/AdminLTE +++ b/AdminLTE @@ -1 +1 @@ -Subproject commit f112893fee8496893687b460c85fefc1dfb88eac +Subproject commit b270c6130b3fc1bb9a221d2ee7178169c64ccac1 diff --git a/AdminLTE_version.txt b/AdminLTE_version.txt index c432e90..c26da17 100644 --- a/AdminLTE_version.txt +++ b/AdminLTE_version.txt @@ -1 +1 @@ -v1.4.2 +v1.4.3.1 diff --git a/alpine.docker b/alpine.docker index a6ab824..083bbee 100644 --- a/alpine.docker +++ b/alpine.docker @@ -13,13 +13,18 @@ RUN apk add --update \ rm -rf /var/cache/apk/* # Customized from submodules -COPY ./alpine/gravity.sh /usr/local/bin/ +COPY ./pi-hole/gravity.sh /usr/local/bin/ +COPY ./pi-hole/advanced/Scripts/* /usr/local/bin/ +RUN sed -i 's|service dnsmasq \(re\)\{0,1\}start|dnsmasq -7 /etc/dnsmasq.d|g' \ + /usr/local/bin/gravity.sh \ + /usr/local/bin/whitelist.sh \ + /usr/local/bin/blacklist.sh COPY ./alpine/nginx.conf /etc/nginx/nginx.conf + # Original upstream pihole code being used COPY ./pi-hole/adlists.default /etc/pihole/ COPY ./pi-hole/adlists.default /etc/.pihole/ COPY ./pi-hole/pihole /usr/local/bin/ -COPY ./pi-hole/advanced/Scripts/* /usr/local/bin/ RUN mkdir -p /opt/ && ln -s /usr/local/bin /opt/pihole COPY ./pi-hole/advanced/dnsmasq.conf.original /etc/dnsmasq.conf COPY ./pi-hole/advanced/01-pihole.conf /etc/dnsmasq.d/ diff --git a/alpine/gravity.sh b/alpine/gravity.sh deleted file mode 100755 index 88922d5..0000000 --- a/alpine/gravity.sh +++ /dev/null @@ -1,359 +0,0 @@ -#!/usr/bin/env bash -# Pi-hole: A black hole for Internet advertisements -# (c) 2015 by Jacob Salmela -# Network-wide ad blocking via your Raspberry Pi -# http://pi-hole.net -# Compiles a list of ad-serving domains by downloading them from multiple sources -# -# Pi-hole is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 2 of the License, or -# (at your option) any later version. - -# Run this script as root or under sudo -echo ":::" - -if [[ $EUID -eq 0 ]];then - echo "::: You are root." -else - echo "::: sudo will be used." - # Check if it is actually installed - # If it isn't, exit because the install cannot complete - if [ -x "$(command -v sudo)" ];then - export SUDO="sudo" - else - echo "::: Please install sudo or run this script as root." - exit 1 - fi -fi - -function helpFunc() -{ - echo "::: Pull in domains from adlists" - echo ":::" - echo "::: Usage: pihole -g" - echo ":::" - echo "::: Options:" - echo "::: -f, --force Force lists to be downloaded, even if they don't need updating." - echo "::: -h, --help Show this help dialog" - exit 1 -} - - -adListFile=/etc/pihole/adlists.list -adListDefault=/etc/pihole/adlists.default -whitelistScript=/opt/pihole/whitelist.sh -blacklistScript=/opt/pihole/blacklist.sh - -#Source the setupVars from install script for the IP -. /etc/pihole/setupVars.conf -#Remove the /* from the end of the IPv4addr. -IPv4addr=${IPv4addr%/*} - -# Variables for various stages of downloading and formatting the list -basename=pihole -piholeDir=/etc/${basename} -adList=${piholeDir}/gravity.list -justDomainsExtension=domains -matterAndLight=${basename}.0.matterandlight.txt -supernova=${basename}.1.supernova.txt -eventHorizon=${basename}.2.eventHorizon.txt -accretionDisc=${basename}.3.accretionDisc.txt - -# After setting defaults, check if there's local overrides -if [[ -r ${piholeDir}/pihole.conf ]];then - echo "::: Local calibration requested..." - . ${piholeDir}/pihole.conf -fi - -########################### -# collapse - begin formation of pihole -function gravity_collapse() { - echo "::: Neutrino emissions detected..." - echo ":::" - #Decide if we're using a custom ad block list, or defaults. - if [ -f ${adListFile} ]; then - #custom file found, use this instead of default - echo -n "::: Custom adList file detected. Reading..." - sources=() - while read -r line; do - #Do not read commented out or blank lines - if [[ ${line} = \#* ]] || [[ ! ${line} ]]; then - echo "" > /dev/null - else - sources+=(${line}) - fi - done < ${adListFile} - echo " done!" - else - #no custom file found, use defaults! - echo -n "::: No custom adlist file detected, reading from default file..." - sources=() - while read -r line; do - #Do not read commented out or blank lines - if [[ ${line} = \#* ]] || [[ ! ${line} ]]; then - echo "" > /dev/null - else - sources+=(${line}) - fi - done < ${adListDefault} - echo " done!" - fi - - # Create the pihole resource directory if it doesn't exist. Future files will be stored here - if [[ -d ${piholeDir} ]];then - # Temporary hack to allow non-root access to pihole directory - # Will update later, needed for existing installs, new installs should - # create this directory as non-root - ${SUDO} chmod 777 ${piholeDir} - echo ":::" - echo "::: Existing pihole directory found" - else - echo "::: Creating pihole directory..." - mkdir ${piholeDir} - ${SUDO} chmod 777 ${piholeDir} - fi -} - -# patternCheck - check to see if curl downloaded any new files. -function gravity_patternCheck() { - patternBuffer=$1 - # check if the patternbuffer is a non-zero length file - if [[ -s "$patternBuffer" ]];then - # Some of the blocklists are copyright, they need to be downloaded - # and stored as is. They can be processed for content after they - # have been saved. - cp "$patternBuffer" "$saveLocation" - echo " List updated, transport successful!" - else - # curl didn't download any host files, probably because of the date check - echo " No changes detected, transport skipped!" - fi -} - -# transport - curl the specified url with any needed command extentions -function gravity_transport() { - url=$1 - cmd_ext=$2 - agent=$3 - - # tmp file, so we don't have to store the (long!) lists in RAM - patternBuffer=$(mktemp) - heisenbergCompensator="" - if [[ -r ${saveLocation} ]]; then - # if domain has been saved, add file for date check to only download newer - heisenbergCompensator="-z $saveLocation" - fi - - # Silently curl url - curl -s -L ${cmd_ext} ${heisenbergCompensator} -A "$agent" ${url} > ${patternBuffer} - # Check for list updates - gravity_patternCheck "$patternBuffer" - # Cleanup - rm -f "$patternBuffer" -} - -# spinup - main gravity function -function gravity_spinup() { - echo ":::" - # Loop through domain list. Download each one and remove commented lines (lines beginning with '# 'or '/') and # blank lines - for ((i = 0; i < "${#sources[@]}"; i++)) - do - url=${sources[$i]} - # Get just the domain from the URL - domain=$(echo "$url" | cut -d'/' -f3) - - # Save the file as list.#.domain - saveLocation=${piholeDir}/list.${i}.${domain}.${justDomainsExtension} - activeDomains[$i]=${saveLocation} - - agent="Mozilla/10.0" - - echo -n "::: Getting $domain list..." - - # Use a case statement to download lists that need special cURL commands - # to complete properly and reset the user agent when required - case "$domain" in - "adblock.mahakala.is") - agent='Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36' - cmd_ext="-e http://forum.xda-developers.com/" - ;; - - "pgl.yoyo.org") - cmd_ext="-d mimetype=plaintext -d hostformat=hosts" - ;; - - # Default is a simple request - *) cmd_ext="" - esac - gravity_transport "$url" "$cmd_ext" "$agent" - done -} - -# Schwarzchild - aggregate domains to one list and add blacklisted domains -function gravity_Schwarzchild() { - echo "::: " - # Find all active domains and compile them into one file and remove CRs - echo -n "::: Aggregating list of domains..." - truncate -s 0 ${piholeDir}/${matterAndLight} - for i in "${activeDomains[@]}" - do - cat "$i" | tr -d '\r' >> ${piholeDir}/${matterAndLight} - done - echo " done!" -} - -function gravity_Blacklist(){ - # Append blacklist entries if they exist - echo -n "::: Running blacklist script to update HOSTS file...." - ${blacklistScript} -f -nr -q > /dev/null - - numBlacklisted=$(wc -l < "/etc/pihole/blacklist.txt") - plural=; [[ "$numBlacklisted" != "1" ]] && plural=s - echo " $numBlacklisted domain${plural} blacklisted!" -} - -function gravity_Whitelist() { - echo ":::" - # Prevent our sources from being pulled into the hole - plural=; [[ "${sources[@]}" != "1" ]] && plural=s - echo -n "::: Adding ${#sources[@]} adlist source${plural} to the whitelist..." - - urls=() - for url in "${sources[@]}" - do - tmp=$(echo "$url" | awk -F '/' '{print $3}') - urls=("${urls[@]}" ${tmp}) - done - echo " done!" - - echo -n "::: Running whitelist script to update HOSTS file...." - ${whitelistScript} -f -nr -q "${urls[@]}" > /dev/null - numWhitelisted=$(wc -l < "/etc/pihole/whitelist.txt") - plural=; [[ "$numWhitelisted" != "1" ]] && plural=s - echo " $numWhitelisted domain${plural} whitelisted!" -} - -function gravity_unique() { - # Sort and remove duplicates - echo -n "::: Removing duplicate domains...." - sort -u ${piholeDir}/${supernova} > ${piholeDir}/${eventHorizon} - echo " done!" - numberOf=$(wc -l < ${piholeDir}/${eventHorizon}) - echo "::: $numberOf unique domains trapped in the event horizon." -} - -function gravity_hostFormat() { - # Format domain list as "192.168.x.x domain.com" - echo "::: Formatting domains into a HOSTS file..." - if [[ -f /etc/hostname ]]; then - hostname=$( ${piholeDir}/${accretionDisc} - cat ${piholeDir}/${eventHorizon} | awk -v ipv4addr="$IPv4addr" -v ipv6addr="$piholeIPv6" '{sub(/\r$/,""); print ipv4addr" "$0"\n"ipv6addr" "$0}' >> ${piholeDir}/${accretionDisc} - else - # Otherwise, just create gravity.list as normal using IPv4 - # Add hostname and dummy domain to the top of gravity.list to make ping result return a friendlier looking domain! Also allows for an easy way to access the Pi-hole admin console (pi.hole/admin) - echo -e "$IPv4addr $hostname\n$IPv4addr pi.hole" > ${piholeDir}/${accretionDisc} - cat ${piholeDir}/${eventHorizon} | awk -v ipv4addr="$IPv4addr" '{sub(/\r$/,""); print ipv4addr" "$0}' >> ${piholeDir}/${accretionDisc} - fi - - # Copy the file over as /etc/pihole/gravity.list so dnsmasq can use it - cp ${piholeDir}/${accretionDisc} ${adList} -} - -# blackbody - remove any remnant files from script processes -function gravity_blackbody() { - # Loop through list files - for file in ${piholeDir}/*.${justDomainsExtension} - do - # If list is in active array then leave it (noop) else rm the list - if [[ " ${activeDomains[@]} " =~ ${file} ]]; then - : - else - rm -f "$file" - fi - done -} - -function gravity_advanced() { - # Remove comments and print only the domain name - # Most of the lists downloaded are already in hosts file format but the spacing/formating is not contigious - # This helps with that and makes it easier to read - # It also helps with debugging so each stage of the script can be researched more in depth - echo -n "::: Formatting list of domains to remove comments...." - #awk '($1 !~ /^#/) { if (NF>1) {print $2} else {print $1}}' ${piholeDir}/${matterAndLight} | sed -nr -e 's/\.{2,}/./g' -e '/\./p' > ${piholeDir}/${supernova} - #Above line does not correctly grab domains where comment is on the same line (e.g 'addomain.com #comment') - #Add additional awk command to read all lines up to a '#', and then continue as we were - cat ${piholeDir}/${matterAndLight} | awk -F'#' '{print $1}' | awk '($1 !~ /^#/) { if (NF>1) {print $2} else {print $1}}' | sed -nr -e 's/\.{2,}/./g' -e '/\./p' > ${piholeDir}/${supernova} - echo " done!" - - numberOf=$(wc -l < ${piholeDir}/${supernova}) - echo "::: $numberOf domains being pulled in by gravity..." - - gravity_unique -} - -function gravity_reload() { - #Clear no longer needed files... - echo ":::" - echo -n "::: Cleaning up un-needed files..." - ${SUDO} rm ${piholeDir}/pihole.*.txt - echo " done!" - - # Reload hosts file - echo ":::" - echo -n "::: Refresh lists in dnsmasq..." - - #ensure /etc/dnsmasq.d/01-pihole.conf is pointing at the correct list! - #First escape forward slashes in the path: - adList=${adList//\//\\\/} - #Now replace the line in dnsmasq file - ${SUDO} sed -i "s/^addn-hosts.*/addn-hosts=$adList/" /etc/dnsmasq.d/01-pihole.conf - dnsmasqPid=$(pidof dnsmasq) - - find "$piholeDir" -type f -exec ${SUDO} chmod 666 {} \; - - if [[ ${dnsmasqPid} ]]; then - # service already running - reload config - ${SUDO} killall -s HUP dnsmasq - else - # service not running, start it up - ${SUDO} dnsmasq -7 /etc/dnsmasq.d - fi - echo " done!" -} - - -for var in "$@" -do - case "$var" in - "-f" | "--force" ) forceGrav=true;; - "-h" | "--help" ) helpFunc;; - esac -done - -if [[ ${forceGrav} == true ]]; then - echo -n "::: Deleting exising list cache..." - ${SUDO} rm /etc/pihole/list.* - echo " done!" -fi - -#Overwrite adlists.default from /etc/.pihole in case any changes have been made. Changes should be saved in /etc/adlists.list -${SUDO} cp /etc/.pihole/adlists.default /etc/pihole/adlists.default -gravity_collapse -gravity_spinup -gravity_Schwarzchild -gravity_advanced -gravity_hostFormat -gravity_blackbody -gravity_Whitelist -gravity_Blacklist -gravity_reload diff --git a/alpine/start.sh b/alpine/start.sh index 209d0c0..4210ba4 100755 --- a/alpine/start.sh +++ b/alpine/start.sh @@ -12,15 +12,7 @@ validate_env setup_saved_variables setup_php_env setup_dnsmasq "$DNS1" "$DNS2" - -# alpine unique currently -ip_versions="IPv4 and IPv6" -if [ "$IPv6" != "True" ] ; then - ip_versions="IPv4" - sed -i '/listen \[::\]:80;/ d' /etc/nginx/nginx.conf -fi; -echo "Using $ip_versions" - +setup_ipv4_ipv6 test_configs test_framework_stubbing diff --git a/common_start.sh b/common_start.sh index cf44941..a890759 100644 --- a/common_start.sh +++ b/common_start.sh @@ -8,8 +8,8 @@ validate_env() { setup_saved_variables() { # /tmp/piholeIP is the current override of auto-lookup in gravity.sh echo "$ServerIP" > /etc/pihole/piholeIP; - echo "IPv4addr=$ServerIP" > /etc/pihole/setupVars.conf; - echo "piholeIPv6=$ServerIPv6" >> /etc/pihole/setupVars.conf; + echo "IPv4_address=$ServerIP" > /etc/pihole/setupVars.conf; + echo "IPv6_address=$ServerIPv6" >> /etc/pihole/setupVars.conf; } setup_dnsmasq() { @@ -26,29 +26,30 @@ setup_dnsmasq() { } setup_php_env() { - if [ ! -f /var/run/dockerpihole-firstboot ] ; then - case $IMAGE in - "debian") setup_php_env_debian ;; - "alpine") setup_php_env_alpine ;; - esac - - touch /var/run/dockerpihole-firstboot - else - echo "Looks like you're restarting this container, skipping php env setup" - fi; + case $IMAGE in + "debian") setup_php_env_debian ;; + "alpine") setup_php_env_alpine ;; + esac } setup_php_env_debian() { - sed -i "/bin-environment/ a\\\t\t\t\"ServerIP\" => \"${ServerIP}\"," $PHP_ENV_CONFIG - sed -i "/bin-environment/ a\\\t\t\t\"PHP_ERROR_LOG\" => \"${PHP_ERROR_LOG}\"," $PHP_ENV_CONFIG - if [ -z "$VIRTUAL_HOST" ] ; then VIRTUAL_HOST="$ServerIP" fi; - sed -i "/bin-environment/ a\\\t\t\t\"VIRTUAL_HOST\" => \"${VIRTUAL_HOST}\"," $PHP_ENV_CONFIG + local vhost_line="\t\t\t\"VIRTUAL_HOST\" => \"${VIRTUAL_HOST}\"," + local serverip_line="\t\t\t\"ServerIP\" => \"${ServerIP}\"," + local php_error_line="\t\t\t\"PHP_ERROR_LOG\" => \"${PHP_ERROR_LOG}\"," + + # idempotent line additions + grep -q "$vhost_line" $PHP_ENV_CONFIG || \ + sed -i "/bin-environment/ a\\${vhost_line}" $PHP_ENV_CONFIG + grep -q "$serverip_line" $PHP_ENV_CONFIG || \ + sed -i "/bin-environment/ a\\${serverip_line}" $PHP_ENV_CONFIG + grep -q "$php_error_line" $PHP_ENV_CONFIG || \ + sed -i "/bin-environment/ a\\${php_error_line}" $PHP_ENV_CONFIG echo "Added ENV to php:" - grep -E '(VIRTUAL_HOST|ServerIP)' $PHP_ENV_CONFIG + grep -E '(VIRTUAL_HOST|ServerIP|PHP_ERROR_LOG)' $PHP_ENV_CONFIG } setup_php_env_alpine() { @@ -66,6 +67,18 @@ setup_php_env_alpine() { cat $PHP_ENV_CONFIG } +setup_ipv4_ipv6() { + local ip_versions="IPv4 and IPv6" + if [ "$IPv6" != "True" ] ; then + ip_versions="IPv4" + case $IMAGE in + "debian") sed -i '/use-ipv6.pl/ d' /etc/lighttpd/lighttpd.conf ;; + "alpine") sed -i '/listen \[::\]:80;/ d' /etc/nginx/nginx.conf ;; + esac + fi; + echo "Using $ip_versions" +} + test_configs() { case $IMAGE in "debian") test_configs_debian ;; @@ -96,5 +109,5 @@ test_configs_alpine() { } test_framework_stubbing() { - if [ -n "$PYTEST" ] ; then sed -i 's/^gravity_spinup/#donotcurl/g' `which gravity.sh`; fi; + if [ -n "$PYTEST" ] ; then sed -i 's/^gravity_spinup$/#donotcurl/g' `which gravity.sh`; fi; } diff --git a/debian-armhf.docker b/debian-armhf.docker index d1f4e8c..32cd750 100644 --- a/debian-armhf.docker +++ b/debian-armhf.docker @@ -29,6 +29,10 @@ COPY ./pi-hole/advanced/pihole.sudo /etc/sudoers.d/pihole COPY ./AdminLTE /var/www/html/admin COPY ./AdminLTE_version.txt /etc/ COPY ./pi-hole_version.txt /etc/ +# Make pihole scripts fail searching for `systemctl`, +# which fails pretty miserably in docker compared to `service` +# For more info see docker/docker issue #7459 +RUN mv `which systemctl` /bin/no_systemctl ENV WEBLOGDIR /var/log/lighttpd RUN mkdir -p /etc/pihole/ && \ @@ -58,6 +62,10 @@ ENV PHP_ERROR_LOG '/var/log/lighttpd/error.log' COPY ./debian/start.sh / COPY ./common_start.sh / +# IPv6 disable flag for networks/devices that do not support it +# not fully supported in debian yet +ENV IPv6 True + EXPOSE 53 53/udp EXPOSE 80 diff --git a/debian.docker b/debian.docker index 55b3cf2..f94c130 100644 --- a/debian.docker +++ b/debian.docker @@ -3,7 +3,7 @@ MAINTAINER adam@diginc.us ENV IMAGE debian -ENV TINI_VERSION v0.9.0 +ENV TINI_VERSION v0.10.0 ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /tini ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini.asc /tini.asc RUN gpg --keyserver ha.pool.sks-keyservers.net --recv-keys 0527A9B7 \ @@ -35,6 +35,10 @@ COPY ./pi-hole/advanced/pihole.sudo /etc/sudoers.d/pihole COPY ./AdminLTE /var/www/html/admin COPY ./AdminLTE_version.txt /etc/ COPY ./pi-hole_version.txt /etc/ +# Make pihole scripts fail searching for `systemctl`, +# which fails pretty miserably in docker compared to `service` +# For more info see docker/docker issue #7459 +RUN mv `which systemctl` /bin/no_systemctl ENV WEBLOGDIR /var/log/lighttpd RUN mkdir -p /etc/pihole/ && \ @@ -64,7 +68,7 @@ ENV PHP_ERROR_LOG '/var/log/lighttpd/error.log' COPY ./debian/start.sh / COPY ./common_start.sh / -# IPV6 disable flag for networks/devices that do not support it +# IPv6 disable flag for networks/devices that do not support it # not fully supported in debian yet ENV IPv6 True diff --git a/debian/start.sh b/debian/start.sh index 3b64ec1..e9d91b2 100755 --- a/debian/start.sh +++ b/debian/start.sh @@ -12,6 +12,7 @@ validate_env setup_saved_variables setup_php_env setup_dnsmasq "$DNS1" "$DNS2" +setup_ipv4_ipv6 test_configs test_framework_stubbing diff --git a/pi-hole b/pi-hole index 5966d76..fc89851 160000 --- a/pi-hole +++ b/pi-hole @@ -1 +1 @@ -Subproject commit 5966d76e9a8ebb5b65327d6ff861cafa86149c7a +Subproject commit fc89851ce95b4ea059ed430e1ad514bdbb3605dd diff --git a/pi-hole_version.txt b/pi-hole_version.txt index 2e8ed09..a2f5086 100644 --- a/pi-hole_version.txt +++ b/pi-hole_version.txt @@ -1 +1 @@ -V2.9.2 +v2.9.4 diff --git a/test/conftest.py b/test/conftest.py index d4db3f3..c687666 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -2,6 +2,7 @@ import pytest import testinfra DEBUG = [] +WEB_SERVER = { 'alpine': 'nginx', 'debian': 'lighttpd' } check_output = testinfra.get_backend( "local://" @@ -44,9 +45,8 @@ def tag(request): return request.param @pytest.fixture -@pytest.mark.parametrize('tag,webserver', [ ( 'alpine', 'nginx' ), ( 'debian', 'lighttpd' ) ]) def webserver(request, tag): - return webserver + return WEB_SERVER[tag] @pytest.fixture() def image(request, tag): @@ -66,8 +66,7 @@ def persist_tag(request): @pytest.fixture(scope='session') def persist_webserver(request, persist_tag): - web_dict = { 'alpine': 'nginx', 'debian': 'lighttpd' } - return web_dict[persist_tag] + return WEB_SERVER[persist_tag] @pytest.fixture(scope='session') def persist_image(request, persist_tag): @@ -102,7 +101,7 @@ def Dig(request): ''' separate container to link to pi-hole and perform lookups ''' ''' a docker pull is faster than running an install of dnsutils ''' def dig(docker_id): - args = '--link {}:pihole'.format(docker_id) + args = '--link {}:test_pihole'.format(docker_id) image = 'azukiapp/dig' cmd = 'tail -f /dev/null' dig_container = DockerGeneric(request, args, image, cmd) diff --git a/test/test_000_build_containers.py b/test/test_000_build_containers.py index 34e90b7..fece246 100644 --- a/test/test_000_build_containers.py +++ b/test/test_000_build_containers.py @@ -9,7 +9,7 @@ run_local = testinfra.get_backend( @pytest.mark.parametrize("upstream,image,tag", [ ( 'alpine:edge', 'alpine.docker', 'diginc/pi-hole:alpine' ), ( 'debian:jessie', 'debian.docker', 'diginc/pi-hole:debian' ), - ( 'jsurf/rpi-raspbian', 'debian-armhf.docker', 'diginc/pi-hole-armhf:debian' ), + ( 'jsurf/rpi-raspbian', 'debian-armhf.docker', 'diginc/pi-hole:arm' ), ]) def test_build_pihole_image(upstream, image, tag): run_local('docker pull {}'.format(upstream)) diff --git a/test/test_start.py b/test/test_start.py index 8027513..8cf18d4 100644 --- a/test/test_start.py +++ b/test/test_start.py @@ -4,6 +4,9 @@ import time ''' Note, testinfra builtins don't seem fully compatible with docker containers (esp. alpine) stripped down nature ''' +WEB_CONFIG = { 'alpine': '/etc/nginx/nginx.conf', 'debian': '/etc/lighttpd/lighttpd.conf' } +IPV6_LINE = { 'alpine': 'listen \[::\]:80', 'debian': 'use-ipv6.pl' } + def test_pihole_default_run_command(Docker): expected_proc = '/sbin/tini -- /start.sh' pgrep = 'pgrep -f "{}" | wc -l || echo 0'.format(expected_proc) @@ -22,10 +25,45 @@ def test_ServerIP_missing_triggers_start_error(Docker): assert start.rc == 1 assert error_msg in start.stdout + +@pytest.mark.parametrize('args,expected_ipv6,expected_stdout', [ + ('-e ServerIP="1.2.3.4"', True, 'IPv4 and IPv6'), + ('-e ServerIP="1.2.3.4" -e "IPv6=True"', True, 'IPv4 and IPv6'), + ('-e ServerIP="1.2.3.4" -e "IPv6=False"', False, 'IPv4'), + ('-e ServerIP="1.2.3.4" -e "IPv6=foobar"', False, 'IPv4'), +]) +@pytest.mark.parametrize('cmd', [ 'tail -f /dev/null' ]) +def test_IPv6_not_True_removes_ipv6(Docker, tag, args, expected_ipv6, expected_stdout): + ''' When a user overrides IPv6=True they only get IPv4 listening webservers ''' + function = Docker.run('. /common_start.sh ; setup_ipv4_ipv6') + assert "Using {}".format(expected_stdout) in function.stdout + ipv6 = Docker.run('grep -q \'{}\' {}'.format(IPV6_LINE[tag], WEB_CONFIG[tag])).rc == 0 + assert ipv6 == expected_ipv6 + +@pytest.mark.parametrize('args, expected_stdout, dns1, dns2', [ + ('-e ServerIP="1.2.3.4"', 'default DNS', '8.8.8.8', '8.8.4.4' ), + ('-e ServerIP="1.2.3.4" -e DNS1="1.2.3.4"', 'custom DNS', '1.2.3.4', '8.8.4.4' ), + ('-e ServerIP="1.2.3.4" -e DNS2="1.2.3.4"', 'custom DNS', '8.8.8.8', '1.2.3.4' ), + ('-e ServerIP="1.2.3.4" -e DNS1="1.2.3.4" -e DNS2="2.2.3.4"', 'custom DNS', '1.2.3.4', '2.2.3.4' ), +]) +@pytest.mark.parametrize('cmd', [ 'tail -f /dev/null' ]) +def test_DNS_Envs_override_defaults(Docker, args, expected_stdout, dns1, dns2): + ''' When DNS environment vars are passed in, they override default dns servers ''' + function = Docker.run('. /common_start.sh ; eval `grep setup_dnsmasq /start.sh`') + assert expected_stdout in function.stdout + + docker_dns_servers = Docker.run('grep "^server=" /etc/dnsmasq.d/01-pihole.conf').stdout + expected_servers = 'server={}\nserver={}\n'.format(dns1, dns2) + assert expected_servers == docker_dns_servers + +''' +Persistent Docker container for testing service post start.sh +''' @pytest.fixture def RunningPiHole(DockerPersist, Slow, persist_webserver): - ''' Persist a working docker-pi-hole to help speed up subsequent tests ''' + ''' Persist a fully started docker-pi-hole to help speed up subsequent tests ''' Slow(lambda: DockerPersist.run('pgrep {}'.format(persist_webserver) ).rc == 0) + Slow(lambda: DockerPersist.run('pgrep dnsmasq').rc == 0) return DockerPersist @pytest.mark.parametrize('hostname,expected_ip', [ @@ -34,7 +72,7 @@ def RunningPiHole(DockerPersist, Slow, persist_webserver): ('b.resolvers.Level3.net', '4.2.2.2') ]) def test_dns_responses(RunningPiHole, hostname, expected_ip): - dig_cmd = "dig +noall +answer {} @pihole | awk '{{ print $5 }}'".format(hostname) + dig_cmd = "dig +time=1 +noall +answer {} @test_pihole | awk '{{ print $5 }}'".format(hostname) lookup = RunningPiHole.dig.run(dig_cmd).stdout.rstrip('\n') assert lookup == expected_ip @@ -68,19 +106,3 @@ def test_admin_requests_load_as_expected(RunningPiHole, ip, url): assert RunningPiHole.run('wc -l /tmp/curled_file ') > 10 assert RunningPiHole.run('grep -q "Content-Security-Policy" /tmp/curled_file ').rc == 0 assert RunningPiHole.run('grep -q "js/pihole/footer.js" /tmp/curled_file ').rc == 0 - -@pytest.mark.parametrize('args, expected_stdout, dns1, dns2', [ - ('-e ServerIP="1.2.3.4"', 'default DNS', '8.8.8.8', '8.8.4.4' ), - ('-e ServerIP="1.2.3.4" -e DNS1="1.2.3.4"', 'custom DNS', '1.2.3.4', '8.8.4.4' ), - ('-e ServerIP="1.2.3.4" -e DNS2="1.2.3.4"', 'custom DNS', '8.8.8.8', '1.2.3.4' ), - ('-e ServerIP="1.2.3.4" -e DNS1="1.2.3.4" -e DNS2="2.2.3.4"', 'custom DNS', '1.2.3.4', '2.2.3.4' ), -]) -@pytest.mark.parametrize('cmd', [ 'tail -f /dev/null' ]) -def test_DNS_Envs_override_defaults(Docker, args, expected_stdout, dns1, dns2): - ''' When DNS environment vars are passed in, they override default dns servers ''' - run_function = Docker.run('. /common_start.sh ; eval `grep setup_dnsmasq /start.sh`') - assert expected_stdout in run_function.stdout - - docker_dns_servers = Docker.run('grep "^server=" /etc/dnsmasq.d/01-pihole.conf').stdout - expected_servers = 'server={}\nserver={}\n'.format(dns1, dns2) - assert expected_servers == docker_dns_servers diff --git a/update.sh b/update.sh index 666d812..0039f53 100755 --- a/update.sh +++ b/update.sh @@ -6,10 +6,6 @@ git submodule foreach git pull origin master; pushd pi-hole ; git describe --tags --abbrev=0 > ../pi-hole_version.txt ; popd ; pushd AdminLTE ; git describe --tags --abbrev=0 > ../AdminLTE_version.txt ; popd ; -# Copy latest gravity and modify to not use `service` command -cp pi-hole/gravity.sh alpine/; -sed -i 's|service dnsmasq start|dnsmasq -7 /etc/dnsmasq.d|g' alpine/gravity.sh - # Copy latest crontab and modify to use docker exec commands cron='./docker-pi-hole.cron' cp -f pi-hole/advanced/pihole.cron ${cron}; From 4e7554b54578ef7c398060a2f66ae8e23c11b54a Mon Sep 17 00:00:00 2001 From: diginc Date: Thu, 20 Oct 2016 23:10:10 -0500 Subject: [PATCH 2/4] Adding a section about upgrading --- README.md | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 6c4e1ce..2bc0236 100644 --- a/README.md +++ b/README.md @@ -82,11 +82,24 @@ As close to the debian image as possible, but cross compiled for ARM architectur Alpine doesn't have an arm cross compileable image at this time. -## Persistence and Customizations +## Upgrading, Persistence, and Customizations The standard pi-hole customization abilities apply to this docker, but with docker twists such as using docker volume mounts to map host stored file configurations over the container defaults. Volumes are also important to persist the configuration incase you have remove the pi-hole container which is a typical docker upgrade pattern. -### Volumes +### Upgrading + +For those unfamilar, the docker way to ugprade is: + +* Throw away your container: `docker rm -f pihole` + * Make sure if you care about your data you have it volume mapped +* Download the latest version of the image: `docker pull diginc/pi-hole` +* Start your container with the newer base image: `docker run ... diginc/pi-hole` (whatever your original run command was) + +Why is this style of upgrading good? A couple reasons: Everyone is starting from the same base which has been tested to known to work. No worrying about upgrading from A to B, B to C, or A to C is required for application developers. + +**If you try to use `pi-hole -up` it will fail**, more about this is covered in [Issue 49](https://github.com/diginc/docker-pi-hole/issues/49) + +### Volumes customizations Here are some relevant wiki pages from pi-hole's documentation and example volume mappings to optionally add to the basic example: @@ -112,7 +125,7 @@ Similarly for the webserver you can customize configs in /etc/nginx (*:alpine* t -## Development[![Build Status](https://travis-ci.org/diginc/docker-pi-hole.svg?branch=dev)](https://travis-ci.org/diginc/docker-pi-hole) +## Development [![Build Status](https://travis-ci.org/diginc/docker-pi-hole.svg?branch=dev)](https://travis-ci.org/diginc/docker-pi-hole) If you plan on making a contribution please pull request to the dev branch. I also build tags of the dev branch for bug fix testing after merges have been made: From e5d09d476b58e16cd5eb761f2d09e93141d5ec11 Mon Sep 17 00:00:00 2001 From: diginc Date: Thu, 20 Oct 2016 23:13:34 -0500 Subject: [PATCH 3/4] clarify --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2bc0236..a24c796 100644 --- a/README.md +++ b/README.md @@ -91,7 +91,7 @@ The standard pi-hole customization abilities apply to this docker, but with dock For those unfamilar, the docker way to ugprade is: * Throw away your container: `docker rm -f pihole` - * Make sure if you care about your data you have it volume mapped + * If you care about your data (logs/customizations), make sure you have it volume mapped or it will be deleted in this step * Download the latest version of the image: `docker pull diginc/pi-hole` * Start your container with the newer base image: `docker run ... diginc/pi-hole` (whatever your original run command was) From 746f0be2e7153bf0f6fdbc8beef48cc5287d5c94 Mon Sep 17 00:00:00 2001 From: Adam Hill Date: Sat, 22 Oct 2016 12:29:28 -0500 Subject: [PATCH 4/4] Update README.md --- README.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index a24c796..1f5b546 100644 --- a/README.md +++ b/README.md @@ -88,16 +88,14 @@ The standard pi-hole customization abilities apply to this docker, but with dock ### Upgrading -For those unfamilar, the docker way to ugprade is: +**If you try to use `pi-hole -up` it will fail.** For those unfamilar, the docker way to ugprade is: * Throw away your container: `docker rm -f pihole` * If you care about your data (logs/customizations), make sure you have it volume mapped or it will be deleted in this step * Download the latest version of the image: `docker pull diginc/pi-hole` * Start your container with the newer base image: `docker run ... diginc/pi-hole` (whatever your original run command was) -Why is this style of upgrading good? A couple reasons: Everyone is starting from the same base which has been tested to known to work. No worrying about upgrading from A to B, B to C, or A to C is required for application developers. - -**If you try to use `pi-hole -up` it will fail**, more about this is covered in [Issue 49](https://github.com/diginc/docker-pi-hole/issues/49) +Why is this style of upgrading good? A couple reasons: Everyone is starting from the same base image which has been tested to know it works. No worrying about upgrading from A to B, B to C, or A to C is required when rolling out updates, reducing complexity. ### Volumes customizations