diff --git a/tests/scripts/source.py b/tests/scripts/source.py
new file mode 100644
index 0000000..2ac0d92
--- /dev/null
+++ b/tests/scripts/source.py
@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+
+import sys
+import subprocess
+
+cmd = [sys.executable, 'speedtest.py', '--source', '127.0.0.1']
+
+p = subprocess.Popen(
+    cmd,
+    stdout=subprocess.PIPE,
+    stderr=subprocess.PIPE
+)
+
+stdout, stderr = p.communicate()
+
+if p.returncode != 1:
+    raise SystemExit('%s did not fail with exit code 1' % ' '.join(cmd))
+
+if b'Invalid argument' not in stderr:
+    raise SystemExit(
+        '"Invalid argument" not found in stderr:\n%s' % stderr.decode()
+    )
diff --git a/tox.ini b/tox.ini
index db53159..4880771 100644
--- a/tox.ini
+++ b/tox.ini
@@ -8,7 +8,7 @@ commands =
     {envpython} -m compileall speedtest.py
     {envpython} speedtest.py
     {envpython} speedtest.py --source 172.17.0.1
-    bash -c \'{envpython} speedtest.py --source 127.0.0.1 | grep "Invalid argument"\'
+    {envpython} tests/scripts/source.py
 
 [testenv:flake8]
 basepython=python
@@ -24,4 +24,4 @@ commands =
     pypy -m compileall speedtest.py
     pypy speedtest.py
     pypy speedtest.py --source 172.17.0.1
-    bash -c \'pypy speedtest.py --source 127.0.0.1 | grep "Invalid argument"\'
+    pypy tests/scripts/source.py