From 643e523f09277454c5f89b51f9c4fc11dcf8dbf2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20R=C3=BChsen?= <tim.ruehsen@gmx.de>
Date: Sun, 27 Sep 2015 19:14:13 +0200
Subject: [PATCH 01/59] Fix psl_builtin_outdated()

---
 src/psl.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/psl.c b/src/psl.c
index e9bcae2..486da89 100644
--- a/src/psl.c
+++ b/src/psl.c
@@ -972,10 +972,10 @@ int psl_builtin_outdated(void)
 {
 	struct stat st;
 
-	if (stat(_psl_filename, &st) == 0 && st.st_mtime > _psl_file_time)
-		return 0;
+	if (stat(_psl_filename, &st) == 0 && st.st_mtime < _psl_file_time)
+		return 1;
 
-	return 1;
+	return 0;
 }
 
 /**

From ac40a6bfc8e577f36d605d392cc46d84448776ce Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20R=C3=BChsen?= <tim.ruehsen@gmx.de>
Date: Sun, 27 Sep 2015 19:14:58 +0200
Subject: [PATCH 02/59] Extend tools/psl --print-info and --version

---
 tools/psl.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tools/psl.c b/tools/psl.c
index 6790efe..fb3972c 100644
--- a/tools/psl.c
+++ b/tools/psl.c
@@ -117,7 +117,7 @@ int main(int argc, const char *const *argv)
 				usage(0, stdout);
 			}
 			else if (!strcmp(*arg, "--version")) {
-				printf("psl %s\n", PACKAGE_VERSION);
+				printf("psl %s (0x%06x)\n", PACKAGE_VERSION, psl_check_version_number(0));
 				printf("libpsl %s\n", psl_get_version());
 				printf("\n");
 				printf("Copyright (C) 2014-2015 Tim Ruehsen\n");
@@ -214,6 +214,7 @@ int main(int argc, const char *const *argv)
 			printf("builtin compile time: %ld (%s)\n", psl_builtin_compile_time(), time2str(psl_builtin_compile_time()));
 			printf("builtin file time: %ld (%s)\n", psl_builtin_file_time(), time2str(psl_builtin_file_time()));
 			printf("builtin SHA1 file hash: %s\n", psl_builtin_sha1sum());
+			printf("builtin outdated: %d\n", psl_builtin_outdated());
 		} else
 			printf("No builtin PSL data available\n");
 	}

From db9a3613d684181738aad4962c47bb7845713157 Mon Sep 17 00:00:00 2001
From: Christopher Meng <i@cicku.me>
Date: Wed, 7 Oct 2015 19:08:26 +0800
Subject: [PATCH 03/59] Correct the present year

---
 NEWS | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/NEWS b/NEWS
index b17e17e..97b270d 100644
--- a/NEWS
+++ b/NEWS
@@ -4,7 +4,7 @@ Copyright (C) 2014-2015 Tim Rühsen
   * Add new function psl_check_version_number()
   * Add version defines to include file
 
-19.09.2025 Release V0.10.0
+19.09.2015 Release V0.10.0
   * Code simplified
   * Less data entries, faster lookups
   * Add new function psl_suffix_wildcard_count()

From dbefdb67678d0774c352aebe4eb37e526c280a3e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20R=C3=BChsen?= <tim.ruehsen@gmx.de>
Date: Thu, 19 Nov 2015 10:06:04 +0100
Subject: [PATCH 04/59] Remove include of bits/stat.h

---
 src/psl.c | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/psl.c b/src/psl.c
index 486da89..ed14a19 100644
--- a/src/psl.c
+++ b/src/psl.c
@@ -87,7 +87,6 @@
 #endif
 
 #include <libpsl.h>
-#include <bits/stat.h>
 
 /* number of elements within an array */
 #define countof(a) (sizeof(a)/sizeof(*(a)))

From 71a3f764140a3a37b924d2ca90c08ec99a1190f9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20R=C3=BChsen?= <tim.ruehsen@gmx.de>
Date: Thu, 19 Nov 2015 10:06:30 +0100
Subject: [PATCH 05/59] Update publixsuffix/list submodule

---
 list | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/list b/list
index 2930bb4..e801df4 160000
--- a/list
+++ b/list
@@ -1 +1 @@
-Subproject commit 2930bb4a5256279e0f7ba44cf9d174fc93ecb732
+Subproject commit e801df4a56ac8c7519d349ad5125433206930d6e

From f3b479fd45d32ac1afb6b5e5cc015c0713751b75 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20R=C3=BChsen?= <tim.ruehsen@gmx.de>
Date: Thu, 19 Nov 2015 11:14:31 +0100
Subject: [PATCH 06/59] Remove -DDATADIR from DEFS in tests/Makefile.am

---
 tests/Makefile.am | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/Makefile.am b/tests/Makefile.am
index 2bc8f82..c71cd99 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -1,4 +1,4 @@
-DEFS = @DEFS@ -DDATADIR=\"$(top_srcdir)/data\" -DSRCDIR=\"$(srcdir)\" -DPSL_FILE=\"$(PSL_FILE)\" -DPSL_TESTFILE=\"$(PSL_TESTFILE)\"
+DEFS = @DEFS@ -DSRCDIR=\"$(srcdir)\" -DPSL_FILE=\"$(PSL_FILE)\" -DPSL_TESTFILE=\"$(PSL_TESTFILE)\"
 AM_CPPFLAGS = -I$(top_srcdir)/include
 LDADD = ../src/libpsl.la
 

From 583f97f2bf7f69ce0428ecc86e4101153ff3b998 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20R=C3=BChsen?= <tim.ruehsen@gmx.de>
Date: Thu, 19 Nov 2015 11:16:16 +0100
Subject: [PATCH 07/59] Copy custom PSL file and test file into distribution
 tarball

---
 Makefile.am | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/Makefile.am b/Makefile.am
index d488ce4..3904754 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -14,4 +14,8 @@ ACLOCAL_AMFLAGS = -I m4 ${ACLOCAL_FLAGS}
 pkgconfigdir = $(libdir)/pkgconfig
 pkgconfig_DATA = libpsl.pc
 
-EXTRA_DIST = config.rpath LICENSE $(PSL_FILE) list/tests/test_psl.txt
+EXTRA_DIST = config.rpath LICENSE
+dist-hook:
+	mkdir -p $(distdir)/list/tests
+	cp -p $(PSL_FILE) $(distdir)/list
+	cp -p $(PSL_TESTFILE) $(distdir)/list/tests

From b53273d4063a748dc017e9b1838745cbe02a848a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20R=C3=BChsen?= <tim.ruehsen@gmx.de>
Date: Thu, 19 Nov 2015 11:18:17 +0100
Subject: [PATCH 08/59] Use absolute PSL path to make psl_builtin_outdated()
 work reliable

---
 src/psl2c.c | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/src/psl2c.c b/src/psl2c.c
index e050194..a050193 100644
--- a/src/psl2c.c
+++ b/src/psl2c.c
@@ -237,6 +237,7 @@ int main(int argc, const char **argv)
 		struct stat st;
 		size_t cmdsize = 16 + strlen(argv[1]);
 		char *cmd = alloca(cmdsize), checksum[64] = "";
+		char *abs_srcfile;
 		const char *source_date_epoch = NULL;
 
 #if 0
@@ -256,6 +257,7 @@ int main(int argc, const char **argv)
 		if (stat(argv[1], &st) != 0)
 			st.st_mtime = 0;
 		fprintf(fpout, "static time_t _psl_file_time = %lu;\n", st.st_mtime);
+
 		if ((source_date_epoch = getenv("SOURCE_DATE_EPOCH")))
 			fprintf(fpout, "static time_t _psl_compile_time = %lu;\n", atol(source_date_epoch));
 		else
@@ -264,7 +266,14 @@ int main(int argc, const char **argv)
 		fprintf(fpout, "static int _psl_nexceptions = %d;\n", psl->nexceptions);
 		fprintf(fpout, "static int _psl_nwildcards = %d;\n", psl->nwildcards);
 		fprintf(fpout, "static const char _psl_sha1_checksum[] = \"%s\";\n", checksum);
-		fprintf(fpout, "static const char _psl_filename[] = \"%s\";\n", argv[1]);
+
+		/* We need an absolute path here, else psl_builtin_outdated() won't work reliable */
+		/* Caveat: symbolic links are resolved by realpath() */
+		if ((abs_srcfile = realpath(argv[1], NULL))) {
+			fprintf(fpout, "static const char _psl_filename[] = \"%s\";\n", abs_srcfile);
+			free(abs_srcfile);
+		} else
+			fprintf(fpout, "static const char _psl_filename[] = \"%s\";\n", argv[1]);
 
 		if (fclose(fpout) != 0)
 			ret = 4;

From d14ada235c4d29b9c54721e78e14bde1a6f79906 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20R=C3=BChsen?= <tim.ruehsen@gmx.de>
Date: Thu, 19 Nov 2015 12:11:27 +0100
Subject: [PATCH 09/59] Use echo without -n in configure.ac (compatibility)

---
 configure.ac | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/configure.ac b/configure.ac
index c3cbd2b..d8572f6 100644
--- a/configure.ac
+++ b/configure.ac
@@ -20,9 +20,9 @@ AC_C_INLINE
 #
 # Generate version defines for include file
 #
-AC_SUBST([LIBPSL_VERSION_MAJOR], [`echo -n $VERSION|cut -d'.' -f1`])
-AC_SUBST([LIBPSL_VERSION_MINOR], [`echo -n $VERSION|cut -d'.' -f2`])
-AC_SUBST([LIBPSL_VERSION_PATCH], [`echo -n $VERSION|cut -d'.' -f3`])
+AC_SUBST([LIBPSL_VERSION_MAJOR], [`echo $VERSION|cut -d'.' -f1`])
+AC_SUBST([LIBPSL_VERSION_MINOR], [`echo $VERSION|cut -d'.' -f2`])
+AC_SUBST([LIBPSL_VERSION_PATCH], [`echo $VERSION|cut -d'.' -f3`])
 AC_SUBST([LIBPSL_VERSION_NUMBER], [`printf '0x%02x%02x%02x' $LIBPSL_VERSION_MAJOR $LIBPSL_VERSION_MINOR $LIBPSL_VERSION_PATCH`])
 AC_CONFIG_FILES([include/libpsl.h])
 

From 519b8c9d17cd701529d6b7b6ed0bcdaaba792c51 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20R=C3=BChsen?= <tim.ruehsen@gmx.de>
Date: Thu, 3 Dec 2015 10:08:04 +0100
Subject: [PATCH 10/59] Add time measurement for test-is-public-all.c

---
 configure.ac               |  5 ++++-
 tests/test-is-public-all.c | 31 ++++++++++++++++++++++++++++++-
 2 files changed, 34 insertions(+), 2 deletions(-)

diff --git a/configure.ac b/configure.ac
index d8572f6..1a8698b 100644
--- a/configure.ac
+++ b/configure.ac
@@ -216,6 +216,9 @@ elif test -n "$NEEDS_NSL" ; then
   LIBS="$LIBS -lnsl"
 fi
 
+# Check for clock_gettime() used for performance measurement
+AC_SEARCH_LIBS(clock_gettime, rt)
+
 # Check for valgrind
 ac_enable_valgrind=no
 AC_ARG_ENABLE(valgrind-tests,
@@ -252,7 +255,7 @@ AC_SUBST(PSL_TESTFILE)
 
 # check for alloca / alloca.h
 AC_FUNC_ALLOCA
-AC_CHECK_FUNCS([strndup])
+AC_CHECK_FUNCS([strndup clock_gettime])
 
 # Override the template file name of the generated .pc file, so that there
 # is no need to rename the template file when the API version changes.
diff --git a/tests/test-is-public-all.c b/tests/test-is-public-all.c
index 705e79f..ea3bb0e 100644
--- a/tests/test-is-public-all.c
+++ b/tests/test-is-public-all.c
@@ -45,6 +45,9 @@
 static int
 	ok,
 	failed;
+#ifdef HAVE_CLOCK_GETTIME
+	struct timespec ts1, ts2;
+#endif
 
 static inline int _isspace_ascii(const char c)
 {
@@ -63,6 +66,10 @@ static void test_psl(void)
 	printf("loaded %d suffixes and %d exceptions\n", psl_suffix_count(psl), psl_suffix_exception_count(psl));
 
 	if ((fp = fopen(PSL_FILE, "r"))) {
+#ifdef HAVE_CLOCK_GETTIME
+		clock_gettime(CLOCK_REALTIME, &ts1);
+#endif
+
 		while ((linep = fgets(buf, sizeof(buf), fp))) {
 			while (_isspace_ascii(*linep)) linep++; /* ignore leading whitespace */
 			if (!*linep) continue; /* skip empty lines */
@@ -107,6 +114,9 @@ static void test_psl(void)
 			}
 		}
 
+#ifdef HAVE_CLOCK_GETTIME
+		clock_gettime(CLOCK_REALTIME, &ts2);
+#endif
 		fclose(fp);
 	} else {
 		printf("Failed to open %s\n", PSL_FILE);
@@ -118,6 +128,10 @@ static void test_psl(void)
 
 int main(int argc, const char * const *argv)
 {
+#ifdef HAVE_CLOCK_GETTIME
+	long ns;
+#endif
+
 	/* if VALGRIND testing is enabled, we have to call ourselves with valgrind checking */
 	if (argc == 1) {
 		const char *valgrind = getenv("TESTS_VALGRIND");
@@ -138,6 +152,21 @@ int main(int argc, const char * const *argv)
 		return 1;
 	}
 
-	printf("Summary: All %d tests passed\n", ok + failed);
+#ifdef HAVE_CLOCK_GETTIME
+	if (ts1.tv_sec == ts2.tv_sec)
+		ns = ts2.tv_nsec - ts1.tv_nsec;
+	else if (ts1.tv_sec == ts2.tv_sec - 1)
+		ns = 1000000000L - (ts2.tv_nsec - ts1.tv_nsec);
+	else
+		ns = 0; /* let's assume something is wrong and skip outputting measured time */
+
+	if (ns)
+		printf("Summary: All %d tests passed in %ld.%06ld ms\n", ok, ns / 1000000, ns % 1000000000);
+	else
+		printf("Summary: All %d tests passed\n", ok);
+#else
+	printf("Summary: All %d tests passed\n", ok);
+#endif
+
 	return 0;
 }

From 7898c22a06cbae1fb9e8ab671905d000e0eea2da Mon Sep 17 00:00:00 2001
From: Darshit Shah <darnir@gmail.com>
Date: Thu, 3 Dec 2015 09:15:23 +0100
Subject: [PATCH 11/59] Fix Shebang line and other issues with autogen.sh

The shebang line was not being interpreted due to a space between # and !. On
fixing it, smaller issues came to the fore due to the effects of the -e switch.
Fixed them all.
---
 autogen.sh | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/autogen.sh b/autogen.sh
index 9de26f5..e714cf8 100755
--- a/autogen.sh
+++ b/autogen.sh
@@ -1,21 +1,21 @@
-# !/bin/sh -e
+#!/bin/sh -e
 
-AUTORECONF=`which autoreconf 2>/dev/null`
+AUTORECONF=$(which autoreconf 2>/dev/null || true)
 if test $? -ne 0; then
   echo "No 'autoreconf' found. You must install the autoconf package."
   exit 1
 fi
 
-GIT=`which git 2>/dev/null`
+GIT=$(which git 2>/dev/null || true)
 if test $? -ne 0; then
   echo "No 'git' found. You must install the git package."
   exit 1
 fi
 
 # create m4 before gtkdocize
-mkdir m4 2>/dev/null
+mkdir m4 2>/dev/null || true
 
-GTKDOCIZE=`which gtkdocize 2>/dev/null`
+GTKDOCIZE=$(which gtkdocize 2>/dev/null || true)
 if test $? -ne 0; then
   echo "No gtk-doc support found. You can't build the docs."
   # rm because gtk-doc.make might be a link to a protected file
@@ -29,7 +29,7 @@ fi
 
 $GIT submodule init
 $GIT submodule update
-$AUTORECONF --install --force --symlink || exit $?
+$AUTORECONF --install --force --symlink
 
 echo
 echo "----------------------------------------------------------------"

From aa0593460ca73a78a83583ddccdf877c6433d11c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20R=C3=BChsen?= <tim.ruehsen@gmx.de>
Date: Fri, 4 Dec 2015 17:15:03 +0100
Subject: [PATCH 12/59] Remove .travis.yml from branch

---
 .travis.yml | 29 -----------------------------
 src/psl2c.c | 48 +++++++++++++++++++++++++++++++++++++++++++-----
 2 files changed, 43 insertions(+), 34 deletions(-)
 delete mode 100644 .travis.yml

diff --git a/.travis.yml b/.travis.yml
deleted file mode 100644
index d72425b..0000000
--- a/.travis.yml
+++ /dev/null
@@ -1,29 +0,0 @@
-language: c
-compiler:
-  - gcc
-  - clang
-# Change this to your needs
-script:
-  - ./autogen.sh
-  - ./configure && make -j4 && make check -j4
-  - ./configure --enable-runtime=libicu --enable-builtin=libicu && make clean && make -j4 && make check -j4
-  - ./configure --enable-runtime=libicu --enable-builtin=libidn2 && make clean && make -j4 && make check -j4
-  - ./configure --enable-runtime=libicu --enable-builtin=libidn && make clean && make -j4 && make check -j4
-  - ./configure --enable-runtime=libicu --disable-builtin && make clean && make -j4 && make check -j4
-  - ./configure --enable-runtime=libidn2 --enable-builtin=libicu && make clean && make -j4 && make check -j4
-  - ./configure --enable-runtime=libidn2 --enable-builtin=libidn2 && make clean && make -j4 && make check -j4
-  - ./configure --enable-runtime=libidn2 --enable-builtin=libidn && make clean && make -j4 && make check -j4
-  - ./configure --enable-runtime=libidn2 --disable-builtin && make clean && make -j4 && make check -j4
-  - ./configure --enable-runtime=libidn --enable-builtin=libicu && make clean && make -j4 && make check -j4
-  - ./configure --enable-runtime=libidn --enable-builtin=libidn2 && make clean && make -j4 && make check -j4
-  - ./configure --enable-runtime=libidn --enable-builtin=libidn && make clean && make -j4 && make check -j4
-  - ./configure --enable-runtime=libidn --disable-builtin && make clean && make -j4 && make check -j4
-  - ./configure --disable-runtime --enable-builtin=libicu && make clean && make -j4 && make check -j4
-  - ./configure --disable-runtime --enable-builtin=libidn2 && make clean && make -j4 && make check -j4
-  - ./configure --disable-runtime --enable-builtin=libidn && make clean && make -j4 && make check -j4
-  - ./configure --disable-runtime --disable-builtin && make clean && make -j4 && make check -j4
-  - ./configure --enable-gtk-doc && make -j4 && make check -j4
-  - make distcheck
-before_install:
-  - sudo apt-get -qq update
-  - sudo apt-get -q install autoconf automake autopoint libtool gtk-doc-tools gettext libidn11 libidn11-dev libidn2-0 libidn2-0-dev libicu48 libicu-dev libunistring0 libunistring-dev
diff --git a/src/psl2c.c b/src/psl2c.c
index a050193..285fee0 100644
--- a/src/psl2c.c
+++ b/src/psl2c.c
@@ -161,6 +161,36 @@ static void _print_psl_entries(FILE *fpout, const _psl_vector_t *v, const char *
 	fprintf(fpout, "};\n");
 }
 
+static void _print_psl_entries_dafsa(FILE *fpout, const _psl_vector_t *v, const char *varname)
+{
+	int it;
+
+#ifdef BUILTIN_GENERATOR_LIBICU
+	do {
+		UVersionInfo version_info;
+		char version[U_MAX_VERSION_STRING_LENGTH];
+
+		u_getVersion(version_info);
+		u_versionToString(version_info, version);
+		fprintf(fpout, "/* automatically generated by psl2c (punycode generated with libicu/%s) */\n", version);
+	} while (0);
+#elif defined(BUILTIN_GENERATOR_LIBIDN2)
+		fprintf(fpout, "/* automatically generated by psl2c (punycode generated with libidn2/%s) */\n", idn2_check_version(NULL));
+#elif defined(BUILTIN_GENERATOR_LIBIDN)
+		fprintf(fpout, "/* automatically generated by psl2c (punycode generated with libidn/%s) */\n", stringprep_check_version(NULL));
+#else
+	fprintf(fpout, "/* automatically generated by psl2c (without punycode support) */\n");
+#endif
+
+	for (it = 0; it < v->cur; it++) {
+		_psl_entry_t *e = _vector_get(v, it);
+
+
+		fprintf(fpout, "\t{ \"%s\", NULL, %hd, %d, %d },\n",
+			e->label_buf, e->length, (int) e->nlabels, (int) e->flags);
+	}
+}
+
 #if 0
 #if !defined(WITH_LIBICU) && !defined(WITH_IDN2)
 static int _str_needs_encoding(const char *s)
@@ -213,17 +243,22 @@ int main(int argc, const char **argv)
 #ifdef _GENERATE_BUILTIN_DATA
 	psl_ctx_t *psl;
 #endif
-	int ret = 0;
+	int ret = 0, argpos = 1, dafsa = 0;
 
-	if (argc != 3) {
-		fprintf(stderr, "Usage: psl2c <infile> <outfile>\n");
+	if (argc == 4 && !strcmp(argv[1], "--dafsa")) {
+		argpos = 2;
+		dafsa = 1;
+	}
+
+	if (argc - argpos != 2) {
+		fprintf(stderr, "Usage: psl2c [--dafsa] <infile> <outfile>\n");
 		fprintf(stderr, "  <infile>  is the 'public_suffix_list.dat', lowercase UTF-8 encoded\n");
 		fprintf(stderr, "  <outfile> is the the C filename to be generated from <infile>\n");
 		return 1;
 	}
 
 #ifdef _GENERATE_BUILTIN_DATA
-	if (!(psl = psl_load_file(argv[1])))
+	if (!(psl = psl_load_file(argv[argpos])))
 		return 2;
 
 	/* look for ambigious or double entries */
@@ -245,7 +280,10 @@ int main(int argc, const char **argv)
 		_add_punycode_if_needed(psl->suffixes);
 #endif
 
-		_print_psl_entries(fpout, psl->suffixes, "suffixes");
+		if (dafsa)
+			_print_psl_entries(fpout, psl->suffixes, "suffixes");
+		else
+			_print_psl_entries_dafsa(fpout, psl->suffixes, "suffixes_dafsa");
 
 		snprintf(cmd, cmdsize, "sha1sum %s", argv[1]);
 		if ((pp = popen(cmd, "r"))) {

From 375aef05ae2e0f1c4edef2ef54b8844dd108b2a4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20R=C3=BChsen?= <tim.ruehsen@gmx.de>
Date: Fri, 4 Dec 2015 17:15:55 +0100
Subject: [PATCH 13/59] Add tools/make_dafsa.py

---
 tools/make_dafsa.py | 469 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 469 insertions(+)
 create mode 100755 tools/make_dafsa.py

diff --git a/tools/make_dafsa.py b/tools/make_dafsa.py
new file mode 100755
index 0000000..78358ef
--- /dev/null
+++ b/tools/make_dafsa.py
@@ -0,0 +1,469 @@
+#!/usr/bin/env python
+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""
+A Deterministic acyclic finite state automaton (DAFSA) is a compact
+representation of an unordered word list (dictionary).
+
+http://en.wikipedia.org/wiki/Deterministic_acyclic_finite_state_automaton
+
+This python program converts a list of strings to a byte array in C++.
+This python program fetches strings and return values from a gperf file
+and generates a C++ file with a byte array representing graph that can be
+used as a memory efficient replacement for the perfect hash table.
+
+The input strings are assumed to consist of printable 7-bit ASCII characters
+and the return values are assumed to be one digit integers.
+
+In this program a DAFSA is a diamond shaped graph starting at a common
+source node and ending at a common sink node. All internal nodes contain
+a label and each word is represented by the labels in one path from
+the source node to the sink node.
+
+The following python represention is used for nodes:
+
+  Source node: [ children ]
+  Internal node: (label, [ children ])
+  Sink node: None
+
+The graph is first compressed by prefixes like a trie. In the next step
+suffixes are compressed so that the graph gets diamond shaped. Finally
+one to one linked nodes are replaced by nodes with the labels joined.
+
+The order of the operations is crucial since lookups will be performed
+starting from the source with no backtracking. Thus a node must have at
+most one child with a label starting by the same character. The output
+is also arranged so that all jumps are to increasing addresses, thus forward
+in memory.
+
+The generated output has suffix free decoding so that the sign of leading
+bits in a link (a reference to a child node) indicate if it has a size of one,
+two or three bytes and if it is the last outgoing link from the actual node.
+A node label is terminated by a byte with the leading bit set.
+
+The generated byte array can described by the following BNF:
+
+<byte> ::= < 8-bit value in range [0x00-0xFF] >
+
+<char> ::= < printable 7-bit ASCII character, byte in range [0x20-0x7F] >
+<end_char> ::= < char + 0x80, byte in range [0xA0-0xFF] >
+<return value> ::= < value + 0x80, byte in range [0x80-0x8F] >
+
+<offset1> ::= < byte in range [0x00-0x3F] >
+<offset2> ::= < byte in range [0x40-0x5F] >
+<offset3> ::= < byte in range [0x60-0x7F] >
+
+<end_offset1> ::= < byte in range [0x80-0xBF] >
+<end_offset2> ::= < byte in range [0xC0-0xDF] >
+<end_offset3> ::= < byte in range [0xE0-0xFF] >
+
+<prefix> ::= <char>
+
+<label> ::= <end_char>
+          | <char> <label>
+
+<end_label> ::= <return_value>
+          | <char> <end_label>
+
+<offset> ::= <offset1>
+           | <offset2> <byte>
+           | <offset3> <byte> <byte>
+
+<end_offset> ::= <end_offset1>
+               | <end_offset2> <byte>
+               | <end_offset3> <byte> <byte>
+
+<offsets> ::= <end_offset>
+            | <offset> <offsets>
+
+<source> ::= <offsets>
+
+<node> ::= <label> <offsets>
+         | <prefix> <node>
+         | <end_label>
+
+<dafsa> ::= <source>
+          | <dafsa> <node>
+
+Decoding:
+
+<char> -> printable 7-bit ASCII character
+<end_char> & 0x7F -> printable 7-bit ASCII character
+<return value> & 0x0F -> integer
+<offset1 & 0x3F> -> integer
+((<offset2> & 0x1F>) << 8) + <byte> -> integer
+((<offset3> & 0x1F>) << 16) + (<byte> << 8) + <byte> -> integer
+
+end_offset1, end_offset2 and and_offset3 are decoded same as offset1,
+offset2 and offset3 respectively.
+
+The first offset in a list of offsets is the distance in bytes between the
+offset itself and the first child node. Subsequent offsets are the distance
+between previous child node and next child node. Thus each offset links a node
+to a child node. The distance is always counted between start addresses, i.e.
+first byte in decoded offset or first byte in child node.
+
+Example 1:
+
+%%
+aa, 1
+a, 2
+%%
+
+The input is first parsed to a list of words:
+["aa1", "a2"]
+
+A fully expanded graph is created from the words:
+source = [node1, node4]
+node1 = ("a", [node2])
+node2 = ("a", [node3])
+node3 = ("\x01", [sink])
+node4 = ("a", [node5])
+node5 = ("\x02", [sink])
+sink = None
+
+Compression results in the following graph:
+source = [node1]
+node1 = ("a", [node2, node3])
+node2 = ("\x02", [sink])
+node3 = ("a\x01", [sink])
+sink = None
+
+A C++ representation of the compressed graph is generated:
+
+const unsigned char dafsa[7] = {
+  0x81, 0xE1, 0x02, 0x81, 0x82, 0x61, 0x81,
+};
+
+The bytes in the generated array has the following meaning:
+
+ 0: 0x81 <end_offset1>  child at position 0 + (0x81 & 0x3F) -> jump to 1
+
+ 1: 0xE1 <end_char>     label character (0xE1 & 0x7F) -> match "a"
+ 2: 0x02 <offset1>      child at position 2 + (0x02 & 0x3F) -> jump to 4
+
+ 3: 0x81 <end_offset1>  child at position 4 + (0x81 & 0x3F) -> jump to 5
+ 4: 0x82 <return_value> 0x82 & 0x0F -> return 2
+
+ 5: 0x61 <char>         label character 0x61 -> match "a"
+ 6: 0x81 <return_value> 0x81 & 0x0F -> return 1
+
+Example 2:
+
+%%
+aa, 1
+bbb, 2
+baa, 1
+%%
+
+The input is first parsed to a list of words:
+["aa1", "bbb2", "baa1"]
+
+Compression results in the following graph:
+source = [node1, node2]
+node1 = ("b", [node2, node3])
+node2 = ("aa\x01", [sink])
+node3 = ("bb\x02", [sink])
+sink = None
+
+A C++ representation of the compressed graph is generated:
+
+const unsigned char dafsa[11] = {
+  0x02, 0x83, 0xE2, 0x02, 0x83, 0x61, 0x61, 0x81, 0x62, 0x62, 0x82,
+};
+
+The bytes in the generated array has the following meaning:
+
+ 0: 0x02 <offset1>      child at position 0 + (0x02 & 0x3F) -> jump to 2
+ 1: 0x83 <end_offset1>  child at position 2 + (0x83 & 0x3F) -> jump to 5
+
+ 2: 0xE2 <end_char>     label character (0xE2 & 0x7F) -> match "b"
+ 3: 0x02 <offset1>      child at position 3 + (0x02 & 0x3F) -> jump to 5
+ 4: 0x83 <end_offset1>  child at position 5 + (0x83 & 0x3F) -> jump to 8
+
+ 5: 0x61 <char>         label character 0x61 -> match "a"
+ 6: 0x61 <char>         label character 0x61 -> match "a"
+ 7: 0x81 <return_value> 0x81 & 0x0F -> return 1
+
+ 8: 0x62 <char>         label character 0x62 -> match "b"
+ 9: 0x62 <char>         label character 0x62 -> match "b"
+10: 0x82 <return_value> 0x82 & 0x0F -> return 2
+"""
+
+import sys
+
+class InputError(Exception):
+  """Exception raised for errors in the input file."""
+
+
+def to_dafsa(words):
+  """Generates a DAFSA from a word list and returns the source node.
+
+  Each word is split into characters so that each character is represented by
+  a unique node. It is assumed the word list is not empty.
+  """
+  if not words:
+    raise InputError('The domain list must not be empty')
+  def ToNodes(word):
+    """Split words into characters"""
+    if not 0x1F < ord(word[0]) < 0x80:
+      raise InputError('Domain names must be printable 7-bit ASCII')
+    if len(word) == 1:
+      return chr(ord(word[0]) & 0x0F), [None]
+    return word[0], [ToNodes(word[1:])]
+  return [ToNodes(word) for word in words]
+
+
+def to_words(node):
+  """Generates a word list from all paths starting from an internal node."""
+  if not node:
+    return ['']
+  return [(node[0] + word) for child in node[1] for word in to_words(child)]
+
+
+def reverse(dafsa):
+  """Generates a new DAFSA that is reversed, so that the old sink node becomes
+  the new source node.
+  """
+  sink = []
+  nodemap = {}
+
+  def dfs(node, parent):
+    """Creates reverse nodes.
+
+    A new reverse node will be created for each old node. The new node will
+    get a reversed label and the parents of the old node as children.
+    """
+    if not node:
+      sink.append(parent)
+    elif id(node) not in nodemap:
+      nodemap[id(node)] = (node[0][::-1], [parent])
+      for child in node[1]:
+        dfs(child, nodemap[id(node)])
+    else:
+      nodemap[id(node)][1].append(parent)
+
+  for node in dafsa:
+    dfs(node, None)
+  return sink
+
+
+def join_labels(dafsa):
+  """Generates a new DAFSA where internal nodes are merged if there is a one to
+  one connection.
+  """
+  parentcount = { id(None): 2 }
+  nodemap = { id(None): None }
+
+  def count_parents(node):
+    """Count incoming references"""
+    if id(node) in parentcount:
+      parentcount[id(node)] += 1
+    else:
+      parentcount[id(node)] = 1
+      for child in node[1]:
+        count_parents(child)
+
+  def join(node):
+    """Create new nodes"""
+    if id(node) not in nodemap:
+      children = [join(child) for child in node[1]]
+      if len(children) == 1 and parentcount[id(node[1][0])] == 1:
+        child = children[0]
+        nodemap[id(node)] = (node[0] + child[0], child[1])
+      else:
+        nodemap[id(node)] = (node[0], children)
+    return nodemap[id(node)]
+
+  for node in dafsa:
+    count_parents(node)
+  return [join(node) for node in dafsa]
+
+
+def join_suffixes(dafsa):
+  """Generates a new DAFSA where nodes that represent the same word lists
+  towards the sink are merged.
+  """
+  nodemap = { frozenset(('',)): None }
+
+  def join(node):
+    """Returns a macthing node. A new node is created if no matching node
+    exists. The graph is accessed in dfs order.
+    """
+    suffixes = frozenset(to_words(node))
+    if suffixes not in nodemap:
+      nodemap[suffixes] = (node[0], [join(child) for child in node[1]])
+    return nodemap[suffixes]
+
+  return [join(node) for node in dafsa]
+
+
+def top_sort(dafsa):
+  """Generates list of nodes in topological sort order."""
+  incoming = {}
+
+  def count_incoming(node):
+    """Counts incoming references."""
+    if node:
+      if id(node) not in incoming:
+        incoming[id(node)] = 1
+        for child in node[1]:
+          count_incoming(child)
+      else:
+        incoming[id(node)] += 1
+
+  for node in dafsa:
+    count_incoming(node)
+
+  for node in dafsa:
+    incoming[id(node)] -= 1
+
+  waiting = [node for node in dafsa if incoming[id(node)] == 0]
+  nodes = []
+
+  while waiting:
+    node = waiting.pop()
+    assert incoming[id(node)] == 0
+    nodes.append(node)
+    for child in node[1]:
+      if child:
+        incoming[id(child)] -= 1
+        if incoming[id(child)] == 0:
+          waiting.append(child)
+  return nodes
+
+
+def encode_links(children, offsets, current):
+  """Encodes a list of children as one, two or three byte offsets."""
+  if not children[0]:
+    # This is an <end_label> node and no links follow such nodes
+    assert len(children) == 1
+    return []
+  guess = 3 * len(children)
+  assert children
+  children = sorted(children, key = lambda x: -offsets[id(x)])
+  while True:
+    offset = current + guess
+    buf = []
+    for child in children:
+      last = len(buf)
+      distance = offset - offsets[id(child)]
+      assert distance > 0 and distance < (1 << 21)
+
+      if distance < (1 << 6):
+        # A 6-bit offset: "s0xxxxxx"
+        buf.append(distance)
+      elif distance < (1 << 13):
+        # A 13-bit offset: "s10xxxxxxxxxxxxx"
+        buf.append(0x40 | (distance >> 8))
+        buf.append(distance & 0xFF)
+      else:
+        # A 21-bit offset: "s11xxxxxxxxxxxxxxxxxxxxx"
+        buf.append(0x60 | (distance >> 16))
+        buf.append((distance >> 8) & 0xFF)
+        buf.append(distance & 0xFF)
+      # Distance in first link is relative to following record.
+      # Distance in other links are relative to previous link.
+      offset -= distance
+    if len(buf) == guess:
+      break
+    guess = len(buf)
+  # Set most significant bit to mark end of links in this node.
+  buf[last] |= (1 << 7)
+  buf.reverse()
+  return buf
+
+
+def encode_prefix(label):
+  """Encodes a node label as a list of bytes without a trailing high byte.
+
+  This method encodes a node if there is exactly one child  and the
+  child follows immidiately after so that no jump is needed. This label
+  will then be a prefix to the label in the child node.
+  """
+  assert label
+  return [ord(c) for c in reversed(label)]
+
+
+def encode_label(label):
+  """Encodes a node label as a list of bytes with a trailing high byte >0x80.
+  """
+  buf = encode_prefix(label)
+  # Set most significant bit to mark end of label in this node.
+  buf[0] |= (1 << 7)
+  return buf
+
+
+def encode(dafsa):
+  """Encodes a DAFSA to a list of bytes"""
+  output = []
+  offsets = {}
+
+  for node in reversed(top_sort(dafsa)):
+    if (len(node[1]) == 1 and node[1][0] and
+        (offsets[id(node[1][0])] == len(output))):
+      output.extend(encode_prefix(node[0]))
+    else:
+      output.extend(encode_links(node[1], offsets, len(output)))
+      output.extend(encode_label(node[0]))
+    offsets[id(node)] = len(output)
+
+  output.extend(encode_links(dafsa, offsets, len(output)))
+  output.reverse()
+  return output
+
+
+def to_cxx(data):
+  """Generates C++ code from a list of encoded bytes."""
+  text = '/* This file is generated. DO NOT EDIT!\n\n'
+  text += 'The byte array encodes effective tld names. See make_dafsa.py for'
+  text += ' documentation.'
+  text += '*/\n\n'
+  text += 'const unsigned char kDafsa[%s] = {\n' % len(data)
+  for i in range(0, len(data), 12):
+    text += '  '
+    text += ', '.join('0x%02x' % byte for byte in data[i:i + 12])
+    text += ',\n'
+  text += '};\n'
+  return text
+
+
+def words_to_cxx(words):
+  """Generates C++ code from a word list"""
+  dafsa = to_dafsa(words)
+  for fun in (reverse, join_suffixes, reverse, join_suffixes, join_labels):
+    dafsa = fun(dafsa)
+  return to_cxx(encode(dafsa))
+
+
+def parse_gperf(infile):
+  """Parses gperf file and extract strings and return code"""
+  lines = [line.strip() for line in infile]
+  # Extract strings after the first '%%' and before the second '%%'.
+  begin = lines.index('%%') + 1
+  end = lines.index('%%', begin)
+  lines = lines[begin:end]
+  for line in lines:
+    if line[-3:-1] != ', ':
+      raise InputError('Expected "domainname, <digit>", found "%s"' % line)
+    # Technically the DAFSA format could support return values in range [0-31],
+    # but the values below are the only with a defined meaning.
+    if line[-1] not in '0124':
+      raise InputError('Expected value to be one of {0,1,2,4}, found "%s"' %
+                       line[-1])
+  return [line[:-3] + line[-1] for line in lines]
+
+
+def main():
+  if len(sys.argv) != 3:
+    print('usage: %s infile outfile' % sys.argv[0])
+    return 1
+  with open(sys.argv[1], 'r') as infile, open(sys.argv[2], 'w') as outfile:
+    outfile.write(words_to_cxx(parse_gperf(infile)))
+  return 0
+
+
+if __name__ == '__main__':
+  sys.exit(main())

From 883e67f0081fedb758f5447382d3e75a26645662 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20R=C3=BChsen?= <tim.ruehsen@gmx.de>
Date: Fri, 4 Dec 2015 21:26:30 +0100
Subject: [PATCH 14/59] Create src/suffixes_dafsa.c with DAFSA C array

---
 list                |  2 +-
 src/Makefile.am     |  1 +
 src/psl2c.c         | 51 +++++++++++++++++++++++++++++++--------------
 tools/make_dafsa.py | 18 +++++++++-------
 4 files changed, 48 insertions(+), 24 deletions(-)

diff --git a/list b/list
index e801df4..c749cdf 160000
--- a/list
+++ b/list
@@ -1 +1 @@
-Subproject commit e801df4a56ac8c7519d349ad5125433206930d6e
+Subproject commit c749cdfe6847c7c299045d160d379117caf47bd3
diff --git a/src/Makefile.am b/src/Makefile.am
index 62cb87d..1111bb3 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -37,3 +37,4 @@ endif
 # PSL_FILE can be set by ./configure --with-psl-file=[PATH]
 suffixes.c: $(PSL_FILE) psl2c$(EXEEXT)
 	./psl2c$(EXEEXT) "$(PSL_FILE)" suffixes.c
+	./psl2c$(EXEEXT) --dafsa "$(PSL_FILE)" suffixes_dafsa.c
\ No newline at end of file
diff --git a/src/psl2c.c b/src/psl2c.c
index 285fee0..33a85f4 100644
--- a/src/psl2c.c
+++ b/src/psl2c.c
@@ -161,8 +161,9 @@ static void _print_psl_entries(FILE *fpout, const _psl_vector_t *v, const char *
 	fprintf(fpout, "};\n");
 }
 
-static void _print_psl_entries_dafsa(FILE *fpout, const _psl_vector_t *v, const char *varname)
+static void _print_psl_entries_dafsa(FILE *fpout, const _psl_vector_t *v)
 {
+	FILE *fp;
 	int it;
 
 #ifdef BUILTIN_GENERATOR_LIBICU
@@ -182,12 +183,30 @@ static void _print_psl_entries_dafsa(FILE *fpout, const _psl_vector_t *v, const
 	fprintf(fpout, "/* automatically generated by psl2c (without punycode support) */\n");
 #endif
 
-	for (it = 0; it < v->cur; it++) {
-		_psl_entry_t *e = _vector_get(v, it);
+	if ((fp = fopen("in.tmp", "w"))) {
+		for (it = 0; it < v->cur; it++) {
+			_psl_entry_t *e = _vector_get(v, it);
+			unsigned char *s = (unsigned char *)e->label_buf;
 
+			/* search for non-ASCII label and skip it */
+			while (*s && *s < 128) s++;
+			if (*s) continue;
 
-		fprintf(fpout, "\t{ \"%s\", NULL, %hd, %d, %d },\n",
-			e->label_buf, e->length, (int) e->nlabels, (int) e->flags);
+			fprintf(fp, "%s, %d\n", e->label_buf, (int) e->flags);
+		}
+
+		fclose(fp);
+	}
+
+	system("../tools/make_dafsa.py in.tmp out.tmp");
+
+	if ((fp = fopen("out.tmp", "r"))) {
+		char buf[256];
+
+		while (fgets(buf, sizeof(buf), fp))
+			fputs(buf, fpout);
+
+		fclose(fp);
 	}
 }
 
@@ -267,10 +286,10 @@ int main(int argc, const char **argv)
 		return 5;
 	}
 */
-	if ((fpout = fopen(argv[2], "w"))) {
+	if ((fpout = fopen(argv[argpos + 1], "w"))) {
 		FILE *pp;
 		struct stat st;
-		size_t cmdsize = 16 + strlen(argv[1]);
+		size_t cmdsize = 16 + strlen(argv[argpos]);
 		char *cmd = alloca(cmdsize), checksum[64] = "";
 		char *abs_srcfile;
 		const char *source_date_epoch = NULL;
@@ -281,18 +300,18 @@ int main(int argc, const char **argv)
 #endif
 
 		if (dafsa)
-			_print_psl_entries(fpout, psl->suffixes, "suffixes");
+			_print_psl_entries_dafsa(fpout, psl->suffixes);
 		else
-			_print_psl_entries_dafsa(fpout, psl->suffixes, "suffixes_dafsa");
+			_print_psl_entries(fpout, psl->suffixes, "suffixes");
 
-		snprintf(cmd, cmdsize, "sha1sum %s", argv[1]);
+		snprintf(cmd, cmdsize, "sha1sum %s", argv[argpos]);
 		if ((pp = popen(cmd, "r"))) {
 			if (fscanf(pp, "%63[0-9a-zA-Z]", checksum) < 1)
 				*checksum = 0;
 			pclose(pp);
 		}
 
-		if (stat(argv[1], &st) != 0)
+		if (stat(argv[argpos], &st) != 0)
 			st.st_mtime = 0;
 		fprintf(fpout, "static time_t _psl_file_time = %lu;\n", st.st_mtime);
 
@@ -307,22 +326,22 @@ int main(int argc, const char **argv)
 
 		/* We need an absolute path here, else psl_builtin_outdated() won't work reliable */
 		/* Caveat: symbolic links are resolved by realpath() */
-		if ((abs_srcfile = realpath(argv[1], NULL))) {
+		if ((abs_srcfile = realpath(argv[argpos], NULL))) {
 			fprintf(fpout, "static const char _psl_filename[] = \"%s\";\n", abs_srcfile);
 			free(abs_srcfile);
 		} else
-			fprintf(fpout, "static const char _psl_filename[] = \"%s\";\n", argv[1]);
+			fprintf(fpout, "static const char _psl_filename[] = \"%s\";\n", argv[argpos]);
 
 		if (fclose(fpout) != 0)
 			ret = 4;
 	} else {
-		fprintf(stderr, "Failed to write open '%s'\n", argv[2]);
+		fprintf(stderr, "Failed to write open '%s'\n", argv[argpos + 1]);
 		ret = 3;
 	}
 
 	psl_free(psl);
 #else
-	if ((fpout = fopen(argv[2], "w"))) {
+	if ((fpout = fopen(argv[argpos + 1], "w"))) {
 		fprintf(fpout, "static _psl_entry_t suffixes[1];\n");
 		fprintf(fpout, "static time_t _psl_file_time;\n");
 		fprintf(fpout, "static time_t _psl_compile_time;\n");
@@ -335,7 +354,7 @@ int main(int argc, const char **argv)
 		if (fclose(fpout) != 0)
 			ret = 4;
 	} else {
-		fprintf(stderr, "Failed to write open '%s'\n", argv[2]);
+		fprintf(stderr, "Failed to write open '%s'\n", argv[argpos + 1]);
 		ret = 3;
 	}
 #endif /* GENERATE_BUILTIN_DATA */
diff --git a/tools/make_dafsa.py b/tools/make_dafsa.py
index 78358ef..6a04bf1 100755
--- a/tools/make_dafsa.py
+++ b/tools/make_dafsa.py
@@ -442,16 +442,16 @@ def parse_gperf(infile):
   """Parses gperf file and extract strings and return code"""
   lines = [line.strip() for line in infile]
   # Extract strings after the first '%%' and before the second '%%'.
-  begin = lines.index('%%') + 1
-  end = lines.index('%%', begin)
-  lines = lines[begin:end]
+  #begin = lines.index('%%') + 1
+  #end = lines.index('%%', begin)
+  #lines = lines[begin:end]
   for line in lines:
     if line[-3:-1] != ', ':
       raise InputError('Expected "domainname, <digit>", found "%s"' % line)
     # Technically the DAFSA format could support return values in range [0-31],
     # but the values below are the only with a defined meaning.
-    if line[-1] not in '0124':
-      raise InputError('Expected value to be one of {0,1,2,4}, found "%s"' %
+    if line[-1] not in '01245':
+      raise InputError('Expected value to be one of {0,1,2,4,5}, found "%s"' %
                        line[-1])
   return [line[:-3] + line[-1] for line in lines]
 
@@ -460,8 +460,12 @@ def main():
   if len(sys.argv) != 3:
     print('usage: %s infile outfile' % sys.argv[0])
     return 1
-  with open(sys.argv[1], 'r') as infile, open(sys.argv[2], 'w') as outfile:
-    outfile.write(words_to_cxx(parse_gperf(infile)))
+  if sys.argv[1] == '-':
+    with open(sys.argv[2], 'w') as outfile:
+      outfile.write(words_to_cxx(parse_gperf(sys.stdin)))
+  else:
+    with open(sys.argv[1], 'r') as infile, open(sys.argv[2], 'w') as outfile:
+      outfile.write(words_to_cxx(parse_gperf(infile)))
   return 0
 
 

From 9d2e93f0b8bba3c602c3c9f7177e9e839c5ee14e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20R=C3=BChsen?= <tim.ruehsen@gmx.de>
Date: Sun, 6 Dec 2015 21:55:56 +0100
Subject: [PATCH 15/59] New function psl_is_public_suffix2()

The current PSL has two sections, ICANN and PRIVATE.
This new function allows to limit the check for one or both
of these sections.
---
 docs/libpsl/libpsl-sections.txt |  1 +
 include/libpsl.h.in             | 27 ++++++++++
 list                            |  2 +-
 src/psl.c                       | 93 +++++++++++++++++++++++++++------
 tests/test-is-public-all.c      | 49 ++++++++++++++++-
 5 files changed, 153 insertions(+), 19 deletions(-)

diff --git a/docs/libpsl/libpsl-sections.txt b/docs/libpsl/libpsl-sections.txt
index a27a9da..643a778 100644
--- a/docs/libpsl/libpsl-sections.txt
+++ b/docs/libpsl/libpsl-sections.txt
@@ -13,6 +13,7 @@ psl_load_fp
 psl_builtin
 psl_free
 psl_is_public_suffix
+psl_is_public_suffix2
 psl_unregistrable_domain
 psl_registrable_domain
 psl_suffix_count
diff --git a/include/libpsl.h.in b/include/libpsl.h.in
index 4f86a50..d4c781b 100644
--- a/include/libpsl.h.in
+++ b/include/libpsl.h.in
@@ -44,6 +44,11 @@
 extern "C" {
 #endif
 
+/* types for psl_is_publix_suffix2() */
+#define PSL_TYPE_ICANN   (1<<0)
+#define PSL_TYPE_PRIVATE (1<<1)
+#define PSL_TYPE_ANY     (PSL_TYPE_ICANN | PSL_TYPE_PRIVATE)
+
 /**
  * psl_error_t:
  * @PSL_SUCCESS: Successful return.
@@ -71,57 +76,79 @@ typedef struct _psl_ctx_st psl_ctx_t;
 /* frees PSL context */
 void
 	psl_free(psl_ctx_t *psl);
+
 /* loads PSL data from file */
 psl_ctx_t *
 	psl_load_file(const char *fname);
+
 /* loads PSL data from FILE pointer */
 psl_ctx_t *
 	psl_load_fp(FILE *fp);
+
 /* retrieves builtin PSL data */
 const psl_ctx_t *
 	psl_builtin(void);
+
 /* checks whether domain is a public suffix or not */
 int
 	psl_is_public_suffix(const psl_ctx_t *psl, const char *domain);
+
+/* checks whether domain is a public suffix regarding the type or not */
+int
+	psl_is_public_suffix2(const psl_ctx_t *psl, const char *domain, int type);
+
 /* checks whether cookie_domain is acceptable for domain or not */
 int
 	psl_is_cookie_domain_acceptable(const psl_ctx_t *psl, const char *hostname, const char *cookie_domain);
+
 /* returns the longest not registrable domain within 'domain' or NULL if none found */
 const char *
 	psl_unregistrable_domain(const psl_ctx_t *psl, const char *domain);
+
 /* returns the shortest possible registrable domain part or NULL if domain is not registrable at all */
 const char *
 	psl_registrable_domain(const psl_ctx_t *psl, const char *domain);
+
 /* convert a string into lowercase UTF-8 */
 psl_error_t
 	psl_str_to_utf8lower(const char *str, const char *encoding, const char *locale, char **lower);
+
 /* does not include exceptions */
 int
 	psl_suffix_count(const psl_ctx_t *psl);
+
 /* just counts exceptions */
 int
 	psl_suffix_exception_count(const psl_ctx_t *psl);
+
 /* just counts wildcards */
 int
 	psl_suffix_wildcard_count(const psl_ctx_t *psl);
+
 /* returns compilation time */
 time_t
 	psl_builtin_compile_time(void);
+
 /* returns mtime of PSL source file */
 time_t
 	psl_builtin_file_time(void);
+
 /* returns SHA1 checksum (hex-encoded, lowercase) of PSL source file */
 const char *
 	psl_builtin_sha1sum(void);
+
 /* returns file name of PSL source file */
 const char *
 	psl_builtin_filename(void);
+
 /* returns library version string */
 const char *
 	psl_get_version(void);
+
 /* checks library version number */
 int
 	psl_check_version_number(int version);
+
 /* returns wether the built-in data is outdated or not */
 int
 	psl_builtin_outdated(void);
diff --git a/list b/list
index e801df4..1f3ad51 160000
--- a/list
+++ b/list
@@ -1 +1 @@
-Subproject commit e801df4a56ac8c7519d349ad5125433206930d6e
+Subproject commit 1f3ad51171235aafe423435606e869f0161582e4
diff --git a/src/psl.c b/src/psl.c
index ed14a19..17717ea 100644
--- a/src/psl.c
+++ b/src/psl.c
@@ -126,9 +126,11 @@ static char *strndup(const char *s, size_t n)
 
 #define countof(a) (sizeof(a)/sizeof(*(a)))
 
-#define _PSL_FLAG_PLAIN     (1<<0)
-#define _PSL_FLAG_EXCEPTION (1<<1)
-#define _PSL_FLAG_WILDCARD  (1<<2)
+#define _PSL_FLAG_EXCEPTION (1<<0)
+#define _PSL_FLAG_WILDCARD  (1<<1)
+#define _PSL_FLAG_ICANN     (1<<2) /* entry of ICANN section */
+#define _PSL_FLAG_PRIVATE   (1<<3) /* entry of PRIVATE section */
+#define _PSL_FLAG_PLAIN     (1<<4) /* just used for PSL syntax checking */
 
 typedef struct {
 	char
@@ -157,6 +159,7 @@ struct _psl_ctx_st {
 	_psl_vector_t
 		*suffixes;
 	int
+		mode,
 		nsuffixes,
 		nexceptions,
 		nwildcards;
@@ -177,7 +180,7 @@ struct _psl_ctx_st {
 	static const char _psl_filename[] = "";
 #endif
 
-/* references to this PSL will result in lookups to built-in data */
+/* references to these PSLs will result in lookups to built-in data */
 static const psl_ctx_t
 	_builtin_psl;
 
@@ -310,10 +313,11 @@ static int _suffix_init(_psl_entry_t *suffix, const char *rule, size_t length)
 	return 0;
 }
 
-static int _psl_is_public_suffix(const psl_ctx_t *psl, const char *domain)
+static int _psl_is_public_suffix(const psl_ctx_t *psl, const char *domain, int type)
 {
 	_psl_entry_t suffix, *rule;
 	const char *p;
+	int builtin;
 
 	/* this function should be called without leading dots, just make sure */
 	suffix.label = domain + (*domain == '.');
@@ -332,7 +336,9 @@ static int _psl_is_public_suffix(const psl_ctx_t *psl, const char *domain)
 	}
 
 	/* if domain has enough labels, it is public */
-	if (psl == &_builtin_psl)
+	builtin = (psl == &_builtin_psl);
+
+	if (builtin)
 		rule = &suffixes[0];
 	else
 		rule = _vector_get(psl->suffixes, 0);
@@ -340,12 +346,18 @@ static int _psl_is_public_suffix(const psl_ctx_t *psl, const char *domain)
 	if (!rule || rule->nlabels < suffix.nlabels - 1)
 		return 0;
 
-	if (psl == &_builtin_psl)
+	if (rule == &suffixes[0])
 		rule = bsearch(&suffix, suffixes, countof(suffixes), sizeof(suffixes[0]), (int(*)(const void *, const void *))_suffix_compare);
 	else
 		rule = _vector_get(psl->suffixes, _vector_find(psl->suffixes, &suffix));
 
 	if (rule) {
+		/* check for correct rule type */
+		if (type == PSL_TYPE_ICANN && !(rule->flags & _PSL_FLAG_ICANN))
+			return 0;
+		else if (type == PSL_TYPE_PRIVATE && !(rule->flags & _PSL_FLAG_PRIVATE))
+			return 0;
+
 		/* definitely a match, no matter if the found rule is a wildcard or not */
 		if (rule->flags & _PSL_FLAG_EXCEPTION)
 			return 0;
@@ -360,12 +372,18 @@ static int _psl_is_public_suffix(const psl_ctx_t *psl, const char *domain)
 		suffix.length = strlen(suffix.label);
 		suffix.nlabels--;
 
-		if (psl == &_builtin_psl)
+		if (builtin)
 			rule = bsearch(&suffix, suffixes, countof(suffixes), sizeof(suffixes[0]), (int(*)(const void *, const void *))_suffix_compare);
 		else
 			rule = _vector_get(psl->suffixes, (pos = _vector_find(psl->suffixes, &suffix)));
 
 		if (rule) {
+			/* check for correct rule type */
+			if (type == PSL_TYPE_ICANN && !(rule->flags & _PSL_FLAG_ICANN))
+				return 0;
+			else if (type == PSL_TYPE_PRIVATE && !(rule->flags & _PSL_FLAG_PRIVATE))
+				return 0;
+
 			if ((rule->flags & _PSL_FLAG_WILDCARD))
 				return 1;
 		}
@@ -399,7 +417,37 @@ int psl_is_public_suffix(const psl_ctx_t *psl, const char *domain)
 	if (!psl || !domain)
 		return 1;
 
-	return _psl_is_public_suffix(psl, domain);
+	return _psl_is_public_suffix(psl, domain, PSL_TYPE_ANY);
+}
+
+/**
+ * psl_is_public_suffix2:
+ * @psl: PSL context
+ * @domain: Domain string
+ * @type: Domain type
+ *
+ * This function checks if @domain is a public suffix by the means of the
+ * [Mozilla Public Suffix List](http://publicsuffix.org).
+ *
+ * @type specifies the PSL section where to perform the lookup. Valid values are
+ * %PSL_TYPE_PRIVATE, %PSL_TYPE_ICANN and %PSL_TYPE_ANY.
+ *
+ * International @domain names have to be either in lowercase UTF-8 or in ASCII form (punycode).
+ * Other encodings result in unexpected behavior.
+ *
+ * @psl is a context returned by either psl_load_file(), psl_load_fp() or
+ * psl_builtin().
+ *
+ * Returns: 1 if domain is a public suffix, 0 if not.
+ *
+ * Since: 0.1
+ */
+int psl_is_public_suffix2(const psl_ctx_t *psl, const char *domain, int type)
+{
+	if (!psl || !domain)
+		return 1;
+
+	return _psl_is_public_suffix(psl, domain, type);
 }
 
 /**
@@ -431,7 +479,7 @@ const char *psl_unregistrable_domain(const psl_ctx_t *psl, const char *domain)
 	 *   'forgot.his.name' and 'name' are in the PSL while 'his.name' is not.
 	 */
 
-	while (!_psl_is_public_suffix(psl, domain)) {
+	while (!_psl_is_public_suffix(psl, domain, 0)) {
 		if ((domain = strchr(domain, '.')))
 			domain++;
 		else
@@ -472,7 +520,7 @@ const char *psl_registrable_domain(const psl_ctx_t *psl, const char *domain)
 	 *   'forgot.his.name' and 'name' are in the PSL while 'his.name' is not.
 	 */
 
-	while (!_psl_is_public_suffix(psl, domain)) {
+	while (!_psl_is_public_suffix(psl, domain, 0)) {
 		if ((p = strchr(domain, '.'))) {
 			regdom = domain;
 			domain = p + 1;
@@ -691,6 +739,7 @@ psl_ctx_t *psl_load_fp(FILE *fp)
 	psl_ctx_t *psl;
 	_psl_entry_t suffix, *suffixp;
 	char buf[256], *linep, *p;
+	int type = 0;
 #ifdef WITH_LIBICU
 	UIDNA *idna;
 	UErrorCode status = 0;
@@ -716,8 +765,20 @@ psl_ctx_t *psl_load_fp(FILE *fp)
 		while (_isspace_ascii(*linep)) linep++; /* ignore leading whitespace */
 		if (!*linep) continue; /* skip empty lines */
 
-		if (*linep == '/' && linep[1] == '/')
+		if (*linep == '/' && linep[1] == '/') {
+			if (!type) {
+				if (strstr(linep + 2, "===BEGIN ICANN DOMAINS==="))
+					type = _PSL_FLAG_ICANN;
+				else if (!type && strstr(linep + 2, "===BEGIN PRIVATE DOMAINS==="))
+					type = _PSL_FLAG_PRIVATE;
+			}
+			else if (type == _PSL_FLAG_ICANN && strstr(linep + 2, "===END ICANN DOMAINS==="))
+				type = 0;
+			else if (type == _PSL_FLAG_PRIVATE && strstr(linep + 2, "===END PRIVATE DOMAINS==="))
+				type = 0;
+
 			continue; /* skip comments */
+		}
 
 		/* parse suffix rule */
 		for (p = linep; *linep && !_isspace_ascii(*linep);) linep++;
@@ -725,7 +786,7 @@ psl_ctx_t *psl_load_fp(FILE *fp)
 
 		if (*p == '!') {
 			p++;
-			suffix.flags = _PSL_FLAG_EXCEPTION;
+			suffix.flags = _PSL_FLAG_EXCEPTION | type;
 			psl->nexceptions++;
 		} else if (*p == '*') {
 			if (*++p != '.') {
@@ -734,13 +795,13 @@ psl_ctx_t *psl_load_fp(FILE *fp)
 			}
 			p++;
 			/* wildcard *.foo.bar implicitely make foo.bar a public suffix */
-			suffix.flags = _PSL_FLAG_WILDCARD | _PSL_FLAG_PLAIN;
+			suffix.flags = _PSL_FLAG_WILDCARD | _PSL_FLAG_PLAIN | type;
 			psl->nwildcards++;
 			psl->nsuffixes++;
 		} else {
 			if (!strchr(p, '.'))
 				continue; /* we do not need an explicit plain TLD rule, already covered by implicit '*' rule */
-			suffix.flags = _PSL_FLAG_PLAIN;
+			suffix.flags = _PSL_FLAG_PLAIN | type;
 			psl->nsuffixes++;
 		}
 
@@ -812,7 +873,7 @@ void psl_free(psl_ctx_t *psl)
  * The builtin data also contains punycode entries, one for each international domain name.
  *
  * If the generation of built-in data has been disabled during compilation, %NULL will be returned.
- * So if using the builtin psl context, you can provide UTF-8 or punycode representations of domains to
+ * When using the builtin psl context, you can provide UTF-8 or punycode representations of domains to
  * functions like psl_is_public_suffix().
  *
  * Returns: Pointer to the built in PSL data or NULL if this data is not available.
diff --git a/tests/test-is-public-all.c b/tests/test-is-public-all.c
index ea3bb0e..b04ccdb 100644
--- a/tests/test-is-public-all.c
+++ b/tests/test-is-public-all.c
@@ -58,7 +58,7 @@ static void test_psl(void)
 {
 	FILE *fp;
 	psl_ctx_t *psl;
-	int result;
+	int result, type = 0;
 	char buf[256], *linep, *p;
 
 	psl = psl_load_file(PSL_FILE); /* PSL_FILE can be set by ./configure --with-psl-file=[PATH] */
@@ -74,8 +74,20 @@ static void test_psl(void)
 			while (_isspace_ascii(*linep)) linep++; /* ignore leading whitespace */
 			if (!*linep) continue; /* skip empty lines */
 
-			if (*linep == '/' && linep[1] == '/')
+			if (*linep == '/' && linep[1] == '/') {
+				if (!type) {
+					if (strstr(linep + 2, "===BEGIN ICANN DOMAINS==="))
+						type = PSL_TYPE_ICANN;
+					else if (!type && strstr(linep + 2, "===BEGIN PRIVATE DOMAINS==="))
+						type = PSL_TYPE_PRIVATE;
+				}
+				else if (type == PSL_TYPE_ICANN && strstr(linep + 2, "===END ICANN DOMAINS==="))
+					type = 0;
+				else if (type == PSL_TYPE_PRIVATE && strstr(linep + 2, "===END PRIVATE DOMAINS==="))
+					type = 0;
+
 				continue; /* skip comments */
+			}
 
 			/* parse suffix rule */
 			for (p = linep; *linep && !_isspace_ascii(*linep);) linep++;
@@ -111,6 +123,39 @@ static void test_psl(void)
 					failed++;
 					printf("psl_is_public_suffix(%s)=%d (expected 1)\n", p, result);
 				} else ok++;
+
+				if (!(strchr(p, '.'))) {
+					/* TLDs are always expected to be Publix Suffixes */
+					if (!(result = psl_is_public_suffix2(psl, p, PSL_TYPE_PRIVATE))) {
+						failed++;
+						printf("psl_is_public_suffix2(%s, PSL_TYPE_PRIVATE)=%d (expected 1)\n", p, result);
+					} else ok++;
+
+					if (!(result = psl_is_public_suffix2(psl, p, PSL_TYPE_ICANN))) {
+						failed++;
+						printf("psl_is_public_suffix2(%s, PSL_TYPE_ICANN)=%d (expected 0)\n", p, result);
+					} else ok++;
+				} else if (type == PSL_TYPE_PRIVATE) {
+					if (!(result = psl_is_public_suffix2(psl, p, PSL_TYPE_PRIVATE))) {
+						failed++;
+						printf("psl_is_public_suffix2(%s, PSL_TYPE_PRIVATE)=%d (expected 1)\n", p, result);
+					} else ok++;
+
+					if ((result = psl_is_public_suffix2(psl, p, PSL_TYPE_ICANN))) {
+						failed++;
+						printf("psl_is_public_suffix2(%s, PSL_TYPE_ICANN)=%d (expected 0)\n", p, result);
+					} else ok++;
+				} else if (type == PSL_TYPE_ICANN) {
+					if (!(result = psl_is_public_suffix2(psl, p, PSL_TYPE_ICANN))) {
+						failed++;
+						printf("psl_is_public_suffix2(%s, PSL_TYPE_ICANN)=%d (expected 1)\n", p, result);
+					} else ok++;
+
+					if ((result = psl_is_public_suffix2(psl, p, PSL_TYPE_PRIVATE))) {
+						failed++;
+						printf("psl_is_public_suffix2(%s, PSL_TYPE_PRIVATE)=%d (expected 0)\n", p, result);
+					} else ok++;
+				}
 			}
 		}
 

From 0ca3741df6a32c48ac940f354928d27fc24e271f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20R=C3=BChsen?= <tim.ruehsen@gmx.de>
Date: Wed, 9 Dec 2015 09:35:04 +0100
Subject: [PATCH 16/59] Use DAWG/DAFSA format for builtin data

This data representation reduces the size of the PSL data
drastically and still allows fast lookups.
---
 AUTHORS                          |   1 +
 README.md                        |   6 +-
 src/LICENSE.chromium             |  30 ++
 src/Makefile.am                  |  19 +-
 src/lookup_string_in_fixed_set.c | 204 ++++++++++++
 {tools => src}/make_dafsa.py     |   6 +-
 src/psl.c                        | 514 +++++++++++++++++--------------
 src/psl2c.c                      |  55 +---
 8 files changed, 552 insertions(+), 283 deletions(-)
 create mode 100644 src/LICENSE.chromium
 create mode 100644 src/lookup_string_in_fixed_set.c
 rename {tools => src}/make_dafsa.py (98%)

diff --git a/AUTHORS b/AUTHORS
index 12e83e6..33dad7b 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -15,3 +15,4 @@ Dagobert Michelsen (Fixed Solaris building)
 Christopher Meng (Fedora building)
 Jakub ÄŒajka
 Giuseppe Scrivano
+Ryan Sleevi (Discussion, Requested DAFSA format and ICANN/PRIVATE support)
diff --git a/README.md b/README.md
index 7bc8fbc..75e1038 100644
--- a/README.md
+++ b/README.md
@@ -14,7 +14,7 @@ Browsers and other web clients can use it to
 
 Libpsl...
 
-- has built-in PSL data for fast access
+- has built-in PSL data for fast access (DAWG/DAFSA reduces size from 180kB to ~32kB)
 - allows to load PSL data from files
 - checks if a given domain is a "public suffix"
 - provides immediate cookie domain verification
@@ -28,6 +28,8 @@ Find more information about the Publix Suffix List [here](http://publicsuffix.or
 
 Download the Public Suffix List [here](https://hg.mozilla.org/mozilla-central/raw-file/tip/netwerk/dns/effective_tld_names.dat).
 
+The DAFSA code has been taken from [Chromium Project](https://code.google.com/p/chromium/).
+
 
 API Documentation
 -----------------
@@ -74,6 +76,8 @@ License
 Libpsl is made available under the terms of the MIT license.<br>
 See the LICENSE file that accompanies this distribution for the full text of the license.
 
+src/make_dafsa.py and src/lookup_string_in_fixed_set.c are licensed under the term written in
+src/LICENSE.chromium.
 
 Building from git
 -----------------
diff --git a/src/LICENSE.chromium b/src/LICENSE.chromium
new file mode 100644
index 0000000..ffe66fe
--- /dev/null
+++ b/src/LICENSE.chromium
@@ -0,0 +1,30 @@
+* The following License is for the source code files
+  make_dafsa.py and lookup_string_in_fixed_set.c.
+
+// Copyright 2015 The Chromium Authors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//    * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//    * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//    * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/src/Makefile.am b/src/Makefile.am
index 1111bb3..9e04d53 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -1,12 +1,12 @@
 # suffixes.c must be created before psl.c is compiled
-BUILT_SOURCES = suffixes.c
+BUILT_SOURCES = suffixes_dafsa.c
 
 # suffixes.c is a built source that must be cleaned
-CLEANFILES = suffixes.c
+CLEANFILES = suffixes_dafsa.c
 
 lib_LTLIBRARIES = libpsl.la
 
-libpsl_la_SOURCES = psl.c
+libpsl_la_SOURCES = psl.c lookup_string_in_fixed_set.c
 libpsl_la_CPPFLAGS = -I$(top_srcdir)/include
 # include ABI version information
 libpsl_la_LDFLAGS = -version-info $(LIBPSL_SO_VERSION)
@@ -21,8 +21,8 @@ if WITH_LIBIDN
 endif
 
 noinst_PROGRAMS = psl2c
-psl2c_SOURCES = psl2c.c
-psl2c_CPPFLAGS = -I$(top_srcdir)/include
+psl2c_SOURCES = psl2c.c lookup_string_in_fixed_set.c
+psl2c_CPPFLAGS = -I$(top_srcdir)/include -DMAKE_DAFSA=\"$(top_srcdir)/src/make_dafsa.py\"
 if BUILTIN_GENERATOR_LIBICU
   psl2c_LDADD = -licuuc
 endif
@@ -33,8 +33,9 @@ if BUILTIN_GENERATOR_LIBIDN
   psl2c_LDADD = @LTLIBICONV@ -lidn -lunistring
 endif
 
-# Build rule for suffix.c
+# Build rule for suffix_dafsa.c
 # PSL_FILE can be set by ./configure --with-psl-file=[PATH]
-suffixes.c: $(PSL_FILE) psl2c$(EXEEXT)
-	./psl2c$(EXEEXT) "$(PSL_FILE)" suffixes.c
-	./psl2c$(EXEEXT) --dafsa "$(PSL_FILE)" suffixes_dafsa.c
\ No newline at end of file
+suffixes_dafsa.c: $(PSL_FILE) psl2c$(EXEEXT)
+	./psl2c$(EXEEXT) "$(PSL_FILE)" suffixes_dafsa.c
+
+EXTRA_DIST = make_dafsa.py LICENSE.chromium
diff --git a/src/lookup_string_in_fixed_set.c b/src/lookup_string_in_fixed_set.c
new file mode 100644
index 0000000..bdec2e5
--- /dev/null
+++ b/src/lookup_string_in_fixed_set.c
@@ -0,0 +1,204 @@
+/* Copyright 2015 The Chromium Authors. All rights reserved.
+ * Use of this source code is governed by a BSD-style license that can be
+ * found in the LICENSE.chromium file.
+ *
+ * Converted to C89 2015 by Tim Rühsen
+ */
+
+#include <stddef.h>
+
+#if defined(__GNUC__) && defined(__GNUC_MINOR__)
+#       define _GCC_VERSION_AT_LEAST(major, minor) ((__GNUC__ > (major)) || (__GNUC__ == (major) && __GNUC_MINOR__ >= (minor)))
+#else
+#       define _GCC_VERSION_AT_LEAST(major, minor) 0
+#endif
+
+#if _GCC_VERSION_AT_LEAST(4,0)
+#  define _HIDDEN __attribute__ ((visibility ("hidden")))
+#else
+#  define _HIDDEN
+#endif
+
+#define CHECK_LT(a, b) if ((a) >= b) return 0
+
+/*
+ * Read next offset from pos.
+ * Returns true if an offset could be read, false otherwise.
+ */
+
+static int GetNextOffset(const unsigned char** pos,
+	const unsigned char* end,
+	const unsigned char** offset)
+{
+	size_t bytes_consumed;
+
+	if (*pos == end)
+		return 0;
+
+	/* When reading an offset the byte array must always contain at least
+	 * three more bytes to consume. First the offset to read, then a node
+	 * to skip over and finally a destination node. No object can be smaller
+	 * than one byte. */
+	CHECK_LT(*pos + 2, end);
+	switch (**pos & 0x60) {
+	case 0x60: /* Read three byte offset */
+		*offset += (((*pos)[0] & 0x1F) << 16) | ((*pos)[1] << 8) | (*pos)[2];
+		bytes_consumed = 3;
+		break;
+	case 0x40: /* Read two byte offset */
+		*offset += (((*pos)[0] & 0x1F) << 8) | (*pos)[1];
+		bytes_consumed = 2;
+		break;
+	default:
+		*offset += (*pos)[0] & 0x3F;
+		bytes_consumed = 1;
+	}
+	if ((**pos & 0x80) != 0) {
+		*pos = end;
+	} else {
+		*pos += bytes_consumed;
+	}
+	return 1;
+}
+
+/*
+ * Check if byte at offset is last in label.
+ */
+
+static int IsEOL(const unsigned char* offset, const unsigned char* end)
+{
+	CHECK_LT(offset, end);
+	return(*offset & 0x80) != 0;
+}
+
+/*
+ * Check if byte at offset matches first character in key.
+ * This version matches characters not last in label.
+ */
+
+static int IsMatch(const unsigned char* offset,
+	const unsigned char* end,
+	const char* key)
+{
+	CHECK_LT(offset, end);
+	return *offset == *key;
+}
+
+/*
+ * Check if byte at offset matches first character in key.
+ * This version matches characters last in label.
+ */
+
+static int IsEndCharMatch(const unsigned char* offset,
+	const unsigned char* end,
+	const char* key)
+{
+	CHECK_LT(offset, end);
+	return *offset == (*key | 0x80);
+}
+
+/*
+ * Read return value at offset.
+ * Returns true if a return value could be read, false otherwise.
+ */
+
+static int GetReturnValue(const unsigned char* offset,
+	const unsigned char* end,
+	int* return_value)
+{
+	CHECK_LT(offset, end);
+	if ((*offset & 0xE0) == 0x80) {
+		*return_value = *offset & 0x0F;
+		return 1;
+	}
+	return 0;
+}
+
+/*
+ *  Looks up the string |key| with length |key_length| in a fixed set of
+ * strings. The set of strings must be known at compile time. It is converted to
+ * a graph structure named a DAFSA (Deterministic Acyclic Finite State
+ * Automaton) by the script make_dafsa.py during compilation. This permits
+ * efficient (in time and space) lookup. The graph generated by make_dafsa.py
+ * takes the form of a constant byte array which should be supplied via the
+ * |graph| and |length| parameters.  The return value is kDafsaNotFound,
+ * kDafsaFound, or a bitmap consisting of one or more of kDafsaExceptionRule,
+ * kDafsaWildcardRule and kDafsaPrivateRule ORed together.
+ * 
+ * Lookup a domain key in a byte array generated by make_dafsa.py.
+ */
+
+/* prototype to skip warning with -Wmissing-prototypes */
+int _HIDDEN LookupStringInFixedSet(const unsigned char*, size_t,const char*, size_t);
+
+int _HIDDEN LookupStringInFixedSet(const unsigned char* graph,
+	size_t length,
+	const char* key,
+	size_t key_length)
+{
+	const unsigned char* pos = graph;
+	const unsigned char* end = graph + length;
+	const unsigned char* offset = pos;
+	const char* key_end = key + key_length;
+
+	while (GetNextOffset(&pos, end, &offset)) {
+		/*char <char>+ end_char offsets
+		 * char <char>+ return value
+		 * char end_char offsets
+		 * char return value
+		 * end_char offsets
+		 * return_value
+		 */
+		int did_consume = 0;
+
+		if (key != key_end && !IsEOL(offset, end)) {
+			/* Leading <char> is not a match. Don't dive into this child */
+			if (!IsMatch(offset, end, key))
+				continue;
+			did_consume = 1;
+			++offset;
+			++key;
+			/* Possible matches at this point:
+			 * <char>+ end_char offsets
+			 * <char>+ return value
+			 * end_char offsets
+			 * return value
+			 */
+
+			/* Remove all remaining <char> nodes possible */
+			while (!IsEOL(offset, end) && key != key_end) {
+				if (!IsMatch(offset, end, key))
+					return -1;
+				++key;
+				++offset;
+			}
+		}
+		/* Possible matches at this point:
+		 * end_char offsets
+		 * return_value
+		 * If one or more <char> elements were consumed, a failure
+		 * to match is terminal. Otherwise, try the next node.
+		 */
+		if (key == key_end) {
+			int return_value;
+
+			if (GetReturnValue(offset, end, &return_value))
+				return return_value;
+			/* The DAFSA guarantees that if the first char is a match, all
+			 * remaining char elements MUST match if the key is truly present.
+			 */
+			if (did_consume)
+				return -1;
+			continue;
+		}
+		if (!IsEndCharMatch(offset, end, key)) {
+			if (did_consume)
+				return -1; /* Unexpected */
+			continue;
+		}
+		++key;
+		pos = ++offset; /* Dive into child */
+	}
+
+	return -1; /* No match */
+}
diff --git a/tools/make_dafsa.py b/src/make_dafsa.py
similarity index 98%
rename from tools/make_dafsa.py
rename to src/make_dafsa.py
index 6a04bf1..8268271 100755
--- a/tools/make_dafsa.py
+++ b/src/make_dafsa.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
+# found in the LICENSE.chromium file.
 
 """
 A Deterministic acyclic finite state automaton (DAFSA) is a compact
@@ -421,7 +421,7 @@ def to_cxx(data):
   text += 'The byte array encodes effective tld names. See make_dafsa.py for'
   text += ' documentation.'
   text += '*/\n\n'
-  text += 'const unsigned char kDafsa[%s] = {\n' % len(data)
+  text += 'static const unsigned char kDafsa[%s] = {\n' % len(data)
   for i in range(0, len(data), 12):
     text += '  '
     text += ', '.join('0x%02x' % byte for byte in data[i:i + 12])
@@ -450,7 +450,7 @@ def parse_gperf(infile):
       raise InputError('Expected "domainname, <digit>", found "%s"' % line)
     # Technically the DAFSA format could support return values in range [0-31],
     # but the values below are the only with a defined meaning.
-    if line[-1] not in '01245':
+    if line[-1] not in '0123456789ABCDEF':
       raise InputError('Expected value to be one of {0,1,2,4,5}, found "%s"' %
                        line[-1])
   return [line[:-3] + line[-1] for line in lines]
diff --git a/src/psl.c b/src/psl.c
index 17717ea..c7c39ef 100644
--- a/src/psl.c
+++ b/src/psl.c
@@ -32,6 +32,18 @@
 # include <config.h>
 #endif
 
+#if defined(__GNUC__) && defined(__GNUC_MINOR__)
+#       define _GCC_VERSION_AT_LEAST(major, minor) ((__GNUC__ > (major)) || (__GNUC__ == (major) && __GNUC_MINOR__ >= (minor)))
+#else
+#       define _GCC_VERSION_AT_LEAST(major, minor) 0
+#endif
+
+#if _GCC_VERSION_AT_LEAST(2,95)
+#  define _UNUSED __attribute__ ((unused))
+#else
+#  define _UNUSED
+#endif
+
 /* if this file is included by psl2c, redefine to use requested library for builtin data */
 #ifdef _LIBPSL_INCLUDED_BY_PSL2C
 #	undef WITH_LIBICU
@@ -167,10 +179,10 @@ struct _psl_ctx_st {
 
 /* include the PSL data compiled by 'psl2c' */
 #ifndef _LIBPSL_INCLUDED_BY_PSL2C
-#	include "suffixes.c"
+#  include "suffixes_dafsa.c"
 #else
 	/* if this source file is included by psl2c.c, provide empty builtin data */
-	static _psl_entry_t suffixes[1];
+	static const unsigned char kDafsa[1];
 	static time_t _psl_file_time;
 	static time_t _psl_compile_time;
 	static int _psl_nsuffixes;
@@ -313,20 +325,196 @@ static int _suffix_init(_psl_entry_t *suffix, const char *rule, size_t length)
 	return 0;
 }
 
+
+static inline int _isspace_ascii(const char c)
+{
+	return c == ' ' || c == '\t' || c == '\r' || c == '\n';
+}
+
+static int _str_is_ascii(const char *s)
+{
+	while (*s && *((unsigned char *)s) < 128) s++;
+
+	return !*s;
+}
+
+#if defined(WITH_LIBIDN)
+/*
+ * Work around a libidn <= 1.30 vulnerability.
+ *
+ * The function checks for a valid UTF-8 character sequence before
+ * passing it to idna_to_ascii_8z().
+ *
+ * [1] http://lists.gnu.org/archive/html/help-libidn/2015-05/msg00002.html
+ * [2] https://lists.gnu.org/archive/html/bug-wget/2015-06/msg00002.html
+ * [3] http://curl.haxx.se/mail/lib-2015-06/0143.html
+ */
+static int _utf8_is_valid(const char *utf8)
+{
+	const unsigned char *s = (const unsigned char *) utf8;
+
+	while (*s) {
+		if ((*s & 0x80) == 0) /* 0xxxxxxx ASCII char */
+			s++;
+		else if ((*s & 0xE0) == 0xC0) /* 110xxxxx 10xxxxxx */ {
+			if ((s[1] & 0xC0) != 0x80)
+				return 0;
+			s += 2;
+		} else if ((*s & 0xF0) == 0xE0) /* 1110xxxx 10xxxxxx 10xxxxxx */ {
+			if ((s[1] & 0xC0) != 0x80 || (s[2] & 0xC0) != 0x80)
+				return 0;
+			s += 3;
+		} else if ((*s & 0xF8) == 0xF0) /* 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx */ {
+			if ((s[1] & 0xC0) != 0x80 || (s[2] & 0xC0) != 0x80 || (s[3] & 0xC0) != 0x80)
+				return 0;
+			s += 4;
+		} else
+			return 0;
+	}
+
+	return 1;
+}
+#endif
+
+typedef void *_psl_idna_t;
+
+static _psl_idna_t *_psl_idna_open(void)
+{
+#if defined(WITH_LIBICU)
+	UErrorCode status = 0;
+	return (void *)uidna_openUTS46(UIDNA_USE_STD3_RULES, &status);
+#endif
+	return NULL;
+}
+
+static void _psl_idna_close(_psl_idna_t *idna _UNUSED)
+{
+#if defined(WITH_LIBICU)
+	if (idna)
+		uidna_close((UIDNA *)idna);
+#endif
+}
+
+static int _psl_idna_toASCII(_psl_idna_t *idna _UNUSED, const char *utf8, char **ascii)
+{
+	int ret = -1;
+
+#if defined(WITH_LIBICU)
+	/* IDNA2008 UTS#46 punycode conversion */
+	if (idna) {
+		char lookupname[128] = "";
+		UErrorCode status = 0;
+		UIDNAInfo info = UIDNA_INFO_INITIALIZER;
+		UChar utf16_dst[128], utf16_src[128];
+		int32_t utf16_src_length;
+
+		u_strFromUTF8(utf16_src, sizeof(utf16_src)/sizeof(utf16_src[0]), &utf16_src_length, utf8, -1, &status);
+		if (U_SUCCESS(status)) {
+			int32_t dst_length = uidna_nameToASCII((UIDNA *)idna, utf16_src, utf16_src_length, utf16_dst, sizeof(utf16_dst)/sizeof(utf16_dst[0]), &info, &status);
+			if (U_SUCCESS(status)) {
+				u_strToUTF8(lookupname, sizeof(lookupname), NULL, utf16_dst, dst_length, &status);
+				if (U_SUCCESS(status)) {
+					if (ascii)
+						*ascii = strdup(lookupname);
+					ret = 0;
+				} /* else
+					fprintf(stderr, "Failed to convert UTF-16 to UTF-8 (status %d)\n", status); */
+			} /* else
+				fprintf(stderr, "Failed to convert to ASCII (status %d)\n", status); */
+		} /* else
+			fprintf(stderr, "Failed to convert UTF-8 to UTF-16 (status %d)\n", status); */
+	}
+#elif defined(WITH_LIBIDN2)
+	int rc;
+	uint8_t *lower, resbuf[256];
+	size_t len = sizeof(resbuf) - 1; /* leave space for additional \0 byte */
+
+	/* we need a conversion to lowercase */
+	lower = u8_tolower((uint8_t *)utf8, u8_strlen((uint8_t *)utf8), 0, UNINORM_NFKC, resbuf, &len);
+	if (!lower) {
+		/* fprintf(stderr, "u8_tolower(%s) failed (%d)\n", utf8, errno); */
+		return -1;
+	}
+
+	/* u8_tolower() does not terminate the result string */
+	if (lower == resbuf) {
+		lower[len]=0;
+	} else {
+		uint8_t *tmp = lower;
+		lower = (uint8_t *)strndup((char *)lower, len);
+		free(tmp);
+	}
+
+	if ((rc = idn2_lookup_u8(lower, (uint8_t **)ascii, 0)) == IDN2_OK) {
+		ret = 0;
+	} /* else
+		fprintf(stderr, "toASCII(%s) failed (%d): %s\n", lower, rc, idn2_strerror(rc)); */
+
+	if (lower != resbuf)
+		free(lower);
+#elif defined(WITH_LIBIDN)
+	int rc;
+
+	if (!_utf8_is_valid(utf8)) {
+		/* fprintf(_(stderr, "Invalid UTF-8 sequence not converted: '%s'\n"), utf8); */
+		return -1;
+	}
+
+	/* idna_to_ascii_8z() automatically converts UTF-8 to lowercase */
+
+	if ((rc = idna_to_ascii_8z(utf8, ascii, IDNA_USE_STD3_ASCII_RULES)) == IDNA_SUCCESS) {
+		ret = 0;
+	} /* else
+		fprintf(_(stderr, "toASCII failed (%d): %s\n"), rc, idna_strerror(rc)); */
+#endif
+
+	return ret;
+}
+
+static void _add_punycode_if_needed(_psl_idna_t *idna, _psl_vector_t *v, _psl_entry_t *e)
+{
+	char *lookupname;
+
+	if (_str_is_ascii(e->label_buf))
+		return;
+
+	if (_psl_idna_toASCII(idna, e->label_buf, &lookupname) == 0) {
+		if (strcmp(e->label_buf, lookupname)) {
+			_psl_entry_t suffix, *suffixp;
+
+			/* fprintf(stderr, "toASCII '%s' -> '%s'\n", e->label_buf, lookupname); */
+			_suffix_init(&suffix, lookupname, strlen(lookupname));
+			suffix.flags = e->flags;
+			suffixp = _vector_get(v, _vector_add(v, &suffix));
+			suffixp->label = suffixp->label_buf; /* set label to changed address */
+		} /* else ignore */
+
+		free(lookupname);
+	}
+}
+
+/* prototype */
+int LookupStringInFixedSet(const unsigned char* graph, size_t length, const char* key, size_t key_length);
+
 static int _psl_is_public_suffix(const psl_ctx_t *psl, const char *domain, int type)
 {
-	_psl_entry_t suffix, *rule;
+	_psl_entry_t suffix;
 	const char *p;
-	int builtin;
+	char *punycode = NULL;
+	int need_conversion = 0;
 
 	/* this function should be called without leading dots, just make sure */
-	suffix.label = domain + (*domain == '.');
-	suffix.length = strlen(suffix.label);
+	if (*domain == '.')
+		domain++;
+
 	suffix.nlabels = 1;
 
-	for (p = suffix.label; *p; p++)
+	for (p = domain; *p; p++) {
 		if (*p == '.')
 			suffix.nlabels++;
+		else if (*((unsigned char *)p) < 128)
+			need_conversion = 1; /* in case domain is non-ascii we need a toASCII conversion */
+	}
 
 	if (suffix.nlabels == 1) {
 		/* TLD, this is the prevailing '*' match.
@@ -335,61 +523,111 @@ static int _psl_is_public_suffix(const psl_ctx_t *psl, const char *domain, int t
 		return 1;
 	}
 
-	/* if domain has enough labels, it is public */
-	builtin = (psl == &_builtin_psl);
+	if (need_conversion) {
+		_psl_idna_t *idna = _psl_idna_open();
 
-	if (builtin)
-		rule = &suffixes[0];
-	else
-		rule = _vector_get(psl->suffixes, 0);
+		if (_psl_idna_toASCII(idna, domain, &punycode) == 0) {
+			suffix.label = punycode;
+			suffix.length = strlen(punycode);
+		} else {
+			/* fallback */
+			suffix.label = domain;
+			suffix.length = p - suffix.label;
+		}
 
-	if (!rule || rule->nlabels < suffix.nlabels - 1)
-		return 0;
-
-	if (rule == &suffixes[0])
-		rule = bsearch(&suffix, suffixes, countof(suffixes), sizeof(suffixes[0]), (int(*)(const void *, const void *))_suffix_compare);
-	else
-		rule = _vector_get(psl->suffixes, _vector_find(psl->suffixes, &suffix));
-
-	if (rule) {
-		/* check for correct rule type */
-		if (type == PSL_TYPE_ICANN && !(rule->flags & _PSL_FLAG_ICANN))
-			return 0;
-		else if (type == PSL_TYPE_PRIVATE && !(rule->flags & _PSL_FLAG_PRIVATE))
-			return 0;
-
-		/* definitely a match, no matter if the found rule is a wildcard or not */
-		if (rule->flags & _PSL_FLAG_EXCEPTION)
-			return 0;
-		if (rule->flags & _PSL_FLAG_PLAIN)
-			return 1;
+		_psl_idna_close(idna);
+	} else {
+		suffix.label = domain;
+		suffix.length = p - suffix.label;
 	}
 
-	if ((suffix.label = strchr(suffix.label, '.'))) {
-		int pos = rule - suffixes;
+	if (psl == &_builtin_psl) {
+		int rc = LookupStringInFixedSet(kDafsa, sizeof(kDafsa), suffix.label, suffix.length);
+		if (rc != -1) {
+			/* check for correct rule type */
+			if (type == PSL_TYPE_ICANN && !(rc & _PSL_FLAG_ICANN))
+				goto suffix_no;
+			else if (type == PSL_TYPE_PRIVATE && !(rc & _PSL_FLAG_PRIVATE))
+				goto suffix_no;
 
-		suffix.label++;
-		suffix.length = strlen(suffix.label);
-		suffix.nlabels--;
+			if (rc & _PSL_FLAG_EXCEPTION)
+				goto suffix_no;
 
-		if (builtin)
-			rule = bsearch(&suffix, suffixes, countof(suffixes), sizeof(suffixes[0]), (int(*)(const void *, const void *))_suffix_compare);
-		else
-			rule = _vector_get(psl->suffixes, (pos = _vector_find(psl->suffixes, &suffix)));
+			/* wildcard *.foo.bar implicitly make foo.bar a public suffix */
+			/* definitely a match, no matter if the found rule is a wildcard or not */
+			goto suffix_yes;
+		}
+		if ((suffix.label = strchr(suffix.label, '.'))) {
+			suffix.label++;
+			suffix.length = strlen(suffix.label);
+			suffix.nlabels--;
+
+			rc = LookupStringInFixedSet(kDafsa, sizeof(kDafsa), suffix.label, suffix.length);
+			if (rc != -1) {
+				/* check for correct rule type */
+				if (type == PSL_TYPE_ICANN && !(rc & _PSL_FLAG_ICANN))
+					goto suffix_no;
+				else if (type == PSL_TYPE_PRIVATE && !(rc & _PSL_FLAG_PRIVATE))
+					goto suffix_no;
+
+				if (rc & _PSL_FLAG_WILDCARD)
+					goto suffix_yes;
+			}
+		}
+	} else {
+		_psl_entry_t *rule = _vector_get(psl->suffixes, 0);
+
+		if (!rule || rule->nlabels < suffix.nlabels - 1)
+			return 0;
+
+		rule = _vector_get(psl->suffixes, _vector_find(psl->suffixes, &suffix));
 
 		if (rule) {
 			/* check for correct rule type */
 			if (type == PSL_TYPE_ICANN && !(rule->flags & _PSL_FLAG_ICANN))
-				return 0;
+				goto suffix_no;
 			else if (type == PSL_TYPE_PRIVATE && !(rule->flags & _PSL_FLAG_PRIVATE))
-				return 0;
+				goto suffix_no;
 
-			if ((rule->flags & _PSL_FLAG_WILDCARD))
-				return 1;
+			if (rule->flags & _PSL_FLAG_EXCEPTION)
+				goto suffix_no;
+
+			/* wildcard *.foo.bar implicitly make foo.bar a public suffix */
+			/* definitely a match, no matter if the found rule is a wildcard or not */
+			goto suffix_yes;
+		}
+
+		if ((suffix.label = strchr(suffix.label, '.'))) {
+			int pos;
+
+			suffix.label++;
+			suffix.length = strlen(suffix.label);
+			suffix.nlabels--;
+
+			rule = _vector_get(psl->suffixes, (pos = _vector_find(psl->suffixes, &suffix)));
+
+			if (rule) {
+				/* check for correct rule type */
+				if (type == PSL_TYPE_ICANN && !(rule->flags & _PSL_FLAG_ICANN))
+					goto suffix_no;
+				else if (type == PSL_TYPE_PRIVATE && !(rule->flags & _PSL_FLAG_PRIVATE))
+					goto suffix_no;
+
+				if (rule->flags & _PSL_FLAG_WILDCARD)
+					goto suffix_yes;
+			}
 		}
 	}
 
+suffix_no:
+	if (punycode)
+		free(punycode);
 	return 0;
+
+suffix_yes:
+	if (punycode)
+		free(punycode);
+	return 1;
 }
 
 /**
@@ -531,167 +769,6 @@ const char *psl_registrable_domain(const psl_ctx_t *psl, const char *domain)
 	return regdom;
 }
 
-static inline int _isspace_ascii(const char c)
-{
-	return c == ' ' || c == '\t' || c == '\r' || c == '\n';
-}
-
-static int _str_is_ascii(const char *s)
-{
-	while (*s && *((unsigned char *)s) < 128) s++;
-
-	return !*s;
-}
-
-#if defined(WITH_LIBIDN)
-/*
- * Work around a libidn <= 1.30 vulnerability.
- *
- * The function checks for a valid UTF-8 character sequence before
- * passing it to idna_to_ascii_8z().
- *
- * [1] http://lists.gnu.org/archive/html/help-libidn/2015-05/msg00002.html
- * [2] https://lists.gnu.org/archive/html/bug-wget/2015-06/msg00002.html
- * [3] http://curl.haxx.se/mail/lib-2015-06/0143.html
- */
-static int _utf8_is_valid(const char *utf8)
-{
-	const unsigned char *s = (const unsigned char *) utf8;
-
-	while (*s) {
-		if ((*s & 0x80) == 0) /* 0xxxxxxx ASCII char */
-			s++;
-		else if ((*s & 0xE0) == 0xC0) /* 110xxxxx 10xxxxxx */ {
-			if ((s[1] & 0xC0) != 0x80)
-				return 0;
-			s += 2;
-		} else if ((*s & 0xF0) == 0xE0) /* 1110xxxx 10xxxxxx 10xxxxxx */ {
-			if ((s[1] & 0xC0) != 0x80 || (s[2] & 0xC0) != 0x80)
-				return 0;
-			s += 3;
-		} else if ((*s & 0xF8) == 0xF0) /* 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx */ {
-			if ((s[1] & 0xC0) != 0x80 || (s[2] & 0xC0) != 0x80 || (s[3] & 0xC0) != 0x80)
-				return 0;
-			s += 4;
-		} else
-			return 0;
-	}
-
-	return 1;
-}
-#endif
-
-#if defined(WITH_LIBICU)
-static void _add_punycode_if_needed(UIDNA *idna, _psl_vector_t *v, _psl_entry_t *e)
-{
-	if (_str_is_ascii(e->label_buf))
-		return;
-
-	/* IDNA2008 UTS#46 punycode conversion */
-	if (idna) {
-		char lookupname[128] = "";
-		UErrorCode status = 0;
-		UIDNAInfo info = UIDNA_INFO_INITIALIZER;
-		UChar utf16_dst[128], utf16_src[128];
-		int32_t utf16_src_length;
-
-		u_strFromUTF8(utf16_src, sizeof(utf16_src)/sizeof(utf16_src[0]), &utf16_src_length, e->label_buf, -1, &status);
-		if (U_SUCCESS(status)) {
-			int32_t dst_length = uidna_nameToASCII(idna, utf16_src, utf16_src_length, utf16_dst, sizeof(utf16_dst)/sizeof(utf16_dst[0]), &info, &status);
-			if (U_SUCCESS(status)) {
-				u_strToUTF8(lookupname, sizeof(lookupname), NULL, utf16_dst, dst_length, &status);
-				if (U_SUCCESS(status)) {
-					if (strcmp(e->label_buf, lookupname)) {
-						_psl_entry_t suffix, *suffixp;
-
-						/* fprintf(stderr, "libicu '%s' -> '%s'\n", e->label_buf, lookupname); */
-						_suffix_init(&suffix, lookupname, strlen(lookupname));
-						suffix.flags = e->flags;
-						suffixp = _vector_get(v, _vector_add(v, &suffix));
-						suffixp->label = suffixp->label_buf; /* set label to changed address */
-					} /* else ignore */
-				} /* else
-					fprintf(stderr, "Failed to convert UTF-16 to UTF-8 (status %d)\n", status); */
-			} /* else
-				fprintf(stderr, "Failed to convert to ASCII (status %d)\n", status); */
-		} /* else
-			fprintf(stderr, "Failed to convert UTF-8 to UTF-16 (status %d)\n", status); */
-	}
-}
-#elif defined(WITH_LIBIDN2)
-static void _add_punycode_if_needed(_psl_vector_t *v, _psl_entry_t *e)
-{
-	char *lookupname = NULL;
-	int rc;
-	uint8_t *lower, resbuf[256];
-	size_t len = sizeof(resbuf) - 1; /* leave space for additional \0 byte */
-
-	if (_str_is_ascii(e->label_buf))
-		return;
-
-	/* we need a conversion to lowercase */
-	lower = u8_tolower((uint8_t *)e->label_buf, u8_strlen((uint8_t *)e->label_buf), 0, UNINORM_NFKC, resbuf, &len);
-	if (!lower) {
-		/* fprintf(stderr, "u8_tolower(%s) failed (%d)\n", e->label_buf, errno); */
-		return;
-	}
-
-	/* u8_tolower() does not terminate the result string */
-	if (lower == resbuf) {
-		lower[len]=0;
-	} else {
-		uint8_t *tmp = lower;
-		lower = (uint8_t *)strndup((char *)lower, len);
-		free(tmp);
-	}
-
-	if ((rc = idn2_lookup_u8(lower, (uint8_t **)&lookupname, 0)) == IDN2_OK) {
-		if (strcmp(e->label_buf, lookupname)) {
-			_psl_entry_t suffix, *suffixp;
-
-			/* fprintf(stderr, "libidn '%s' -> '%s'\n", e->label_buf, lookupname); */
-			_suffix_init(&suffix, lookupname, strlen(lookupname));
-			suffix.flags = e->flags;
-			suffixp = _vector_get(v, _vector_add(v, &suffix));
-			suffixp->label = suffixp->label_buf; /* set label to changed address */
-		} /* else ignore */
-	} /* else
-		fprintf(stderr, "toASCII(%s) failed (%d): %s\n", lower, rc, idn2_strerror(rc)); */
-
-	if (lower != resbuf)
-		free(lower);
-}
-#elif defined(WITH_LIBIDN)
-static void _add_punycode_if_needed(_psl_vector_t *v, _psl_entry_t *e)
-{
-	char *lookupname = NULL;
-	int rc;
-
-	if (_str_is_ascii(e->label_buf))
-		return;
-
-	if (!_utf8_is_valid(e->label_buf)) {
-		/* fprintf(_(stderr, "Invalid UTF-8 sequence not converted: '%s'\n"), e->label_buf); */
-		return;
-	}
-
-	/* idna_to_ascii_8z() automatically converts UTF-8 to lowercase */
-
-	if ((rc = idna_to_ascii_8z(e->label_buf, &lookupname, IDNA_USE_STD3_ASCII_RULES)) == IDNA_SUCCESS) {
-		if (strcmp(e->label_buf, lookupname)) {
-			_psl_entry_t suffix, *suffixp;
-
-			/* fprintf(stderr, "libidn '%s' -> '%s'\n", e->label_buf, lookupname); */
-			_suffix_init(&suffix, lookupname, strlen(lookupname));
-			suffix.flags = e->flags;
-			suffixp = _vector_get(v, _vector_add(v, &suffix));
-			suffixp->label = suffixp->label_buf; /* set label to changed address */
-		} /* else ignore */
-	} /* else
-		fprintf(_(stderr, "toASCII failed (%d): %s\n"), rc, idna_strerror(rc)); */
-}
-#endif
-
 /**
  * psl_load_file:
  * @fname: Name of PSL file
@@ -740,10 +817,7 @@ psl_ctx_t *psl_load_fp(FILE *fp)
 	_psl_entry_t suffix, *suffixp;
 	char buf[256], *linep, *p;
 	int type = 0;
-#ifdef WITH_LIBICU
-	UIDNA *idna;
-	UErrorCode status = 0;
-#endif
+	_psl_idna_t *idna;
 
 	if (!fp)
 		return NULL;
@@ -751,9 +825,7 @@ psl_ctx_t *psl_load_fp(FILE *fp)
 	if (!(psl = calloc(1, sizeof(psl_ctx_t))))
 		return NULL;
 
-#ifdef WITH_LIBICU
-	idna = uidna_openUTS46(UIDNA_USE_STD3_RULES, &status);
-#endif
+	idna = _psl_idna_open();
 
 	/*
 	 *  as of 02.11.2012, the list at http://publicsuffix.org/list/ contains ~6000 rules and 40 exceptions.
@@ -794,7 +866,7 @@ psl_ctx_t *psl_load_fp(FILE *fp)
 				continue;
 			}
 			p++;
-			/* wildcard *.foo.bar implicitely make foo.bar a public suffix */
+			/* wildcard *.foo.bar implicitly make foo.bar a public suffix */
 			suffix.flags = _PSL_FLAG_WILDCARD | _PSL_FLAG_PLAIN | type;
 			psl->nwildcards++;
 			psl->nsuffixes++;
@@ -829,20 +901,14 @@ psl_ctx_t *psl_load_fp(FILE *fp)
 			}
 
 			suffixp->label = suffixp->label_buf; /* set label to changed address */
-#ifdef WITH_LIBICU
+
 			_add_punycode_if_needed(idna, psl->suffixes, suffixp);
-#elif defined(WITH_LIBIDN2) || defined(WITH_LIBIDN)
-			_add_punycode_if_needed(psl->suffixes, suffixp);
-#endif
 		}
 	}
 
 	_vector_sort(psl->suffixes);
 
-#ifdef WITH_LIBICU
-	if (idna)
-		uidna_close(idna);
-#endif
+	_psl_idna_close(idna);
 
 	return psl;
 }
@@ -1184,7 +1250,7 @@ int psl_is_cookie_domain_acceptable(const psl_ctx_t *psl, const char *hostname,
  *
  * Since: 0.4
  */
-psl_error_t psl_str_to_utf8lower(const char *str, const char *encoding, const char *locale, char **lower)
+psl_error_t psl_str_to_utf8lower(const char *str, const char *encoding, const char *locale _UNUSED, char **lower)
 {
 	int ret = PSL_ERR_INVALID_ARG;
 
diff --git a/src/psl2c.c b/src/psl2c.c
index 33a85f4..7d02679 100644
--- a/src/psl2c.c
+++ b/src/psl2c.c
@@ -128,39 +128,6 @@ static int _check_psl(const psl_ctx_t *psl)
 }
 #endif
 
-static void _print_psl_entries(FILE *fpout, const _psl_vector_t *v, const char *varname)
-{
-	int it;
-
-#ifdef BUILTIN_GENERATOR_LIBICU
-	do {
-		UVersionInfo version_info;
-		char version[U_MAX_VERSION_STRING_LENGTH];
-
-		u_getVersion(version_info);
-		u_versionToString(version_info, version);
-		fprintf(fpout, "/* automatically generated by psl2c (punycode generated with libicu/%s) */\n", version);
-	} while (0);
-#elif defined(BUILTIN_GENERATOR_LIBIDN2)
-		fprintf(fpout, "/* automatically generated by psl2c (punycode generated with libidn2/%s) */\n", idn2_check_version(NULL));
-#elif defined(BUILTIN_GENERATOR_LIBIDN)
-		fprintf(fpout, "/* automatically generated by psl2c (punycode generated with libidn/%s) */\n", stringprep_check_version(NULL));
-#else
-	fprintf(fpout, "/* automatically generated by psl2c (without punycode support) */\n");
-#endif
-
-	fprintf(fpout, "static _psl_entry_t %s[] = {\n", varname);
-
-	for (it = 0; it < v->cur; it++) {
-		_psl_entry_t *e = _vector_get(v, it);
-
-		fprintf(fpout, "\t{ \"%s\", NULL, %hd, %d, %d },\n",
-			e->label_buf, e->length, (int) e->nlabels, (int) e->flags);
-	}
-
-	fprintf(fpout, "};\n");
-}
-
 static void _print_psl_entries_dafsa(FILE *fpout, const _psl_vector_t *v)
 {
 	FILE *fp;
@@ -192,13 +159,14 @@ static void _print_psl_entries_dafsa(FILE *fpout, const _psl_vector_t *v)
 			while (*s && *s < 128) s++;
 			if (*s) continue;
 
-			fprintf(fp, "%s, %d\n", e->label_buf, (int) e->flags);
+			fprintf(fp, "%s, %X\n", e->label_buf, (int) (e->flags & 0x0F));
 		}
 
 		fclose(fp);
 	}
 
-	system("../tools/make_dafsa.py in.tmp out.tmp");
+	if ((it = system(MAKE_DAFSA " in.tmp out.tmp")))
+		fprintf(stderr, "Failed to execute " MAKE_DAFSA "\n");
 
 	if ((fp = fopen("out.tmp", "r"))) {
 		char buf[256];
@@ -208,6 +176,9 @@ static void _print_psl_entries_dafsa(FILE *fpout, const _psl_vector_t *v)
 
 		fclose(fp);
 	}
+
+	unlink("in.tmp");
+	unlink("out.tmp");
 }
 
 #if 0
@@ -262,15 +233,10 @@ int main(int argc, const char **argv)
 #ifdef _GENERATE_BUILTIN_DATA
 	psl_ctx_t *psl;
 #endif
-	int ret = 0, argpos = 1, dafsa = 0;
-
-	if (argc == 4 && !strcmp(argv[1], "--dafsa")) {
-		argpos = 2;
-		dafsa = 1;
-	}
+	int ret = 0, argpos = 1;
 
 	if (argc - argpos != 2) {
-		fprintf(stderr, "Usage: psl2c [--dafsa] <infile> <outfile>\n");
+		fprintf(stderr, "Usage: psl2c <infile> <outfile>\n");
 		fprintf(stderr, "  <infile>  is the 'public_suffix_list.dat', lowercase UTF-8 encoded\n");
 		fprintf(stderr, "  <outfile> is the the C filename to be generated from <infile>\n");
 		return 1;
@@ -299,10 +265,7 @@ int main(int argc, const char **argv)
 		_add_punycode_if_needed(psl->suffixes);
 #endif
 
-		if (dafsa)
-			_print_psl_entries_dafsa(fpout, psl->suffixes);
-		else
-			_print_psl_entries(fpout, psl->suffixes, "suffixes");
+		_print_psl_entries_dafsa(fpout, psl->suffixes);
 
 		snprintf(cmd, cmdsize, "sha1sum %s", argv[argpos]);
 		if ((pp = popen(cmd, "r"))) {

From df053f91cca36f47d4b994938dbf2bf4f84b6e8e Mon Sep 17 00:00:00 2001
From: Darshit Shah <darnir@gmail.com>
Date: Thu, 10 Dec 2015 23:16:37 +0100
Subject: [PATCH 17/59] Faster, more concurrent travis builds

---
 .travis.yml | 52 ++++++++++++++++++++++++++++++++--------------------
 1 file changed, 32 insertions(+), 20 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index d72425b..3a0823c 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,29 +1,41 @@
+sudo: false
+
 language: c
+
 compiler:
   - gcc
   - clang
-# Change this to your needs
+
+env:
+    - RUNTIME=libicu
+    - RUNTIME=libidn2
+    - RUNTIME=libidn
+    - RUNTIME=no
+
+addons:
+    apt:
+        packages:
+            - automake
+            - autoconf
+            - autopoint
+            - libtool
+            - gtk-doc-tools
+            - gettext
+            - libidn11
+            - libidn11-dev
+            - libidn2-0
+            - libidn2-0-dev
+            - libicu48
+            - libicu-dev
+            - libunistring0
+            - libunistring-dev
+
 script:
   - ./autogen.sh
   - ./configure && make -j4 && make check -j4
-  - ./configure --enable-runtime=libicu --enable-builtin=libicu && make clean && make -j4 && make check -j4
-  - ./configure --enable-runtime=libicu --enable-builtin=libidn2 && make clean && make -j4 && make check -j4
-  - ./configure --enable-runtime=libicu --enable-builtin=libidn && make clean && make -j4 && make check -j4
-  - ./configure --enable-runtime=libicu --disable-builtin && make clean && make -j4 && make check -j4
-  - ./configure --enable-runtime=libidn2 --enable-builtin=libicu && make clean && make -j4 && make check -j4
-  - ./configure --enable-runtime=libidn2 --enable-builtin=libidn2 && make clean && make -j4 && make check -j4
-  - ./configure --enable-runtime=libidn2 --enable-builtin=libidn && make clean && make -j4 && make check -j4
-  - ./configure --enable-runtime=libidn2 --disable-builtin && make clean && make -j4 && make check -j4
-  - ./configure --enable-runtime=libidn --enable-builtin=libicu && make clean && make -j4 && make check -j4
-  - ./configure --enable-runtime=libidn --enable-builtin=libidn2 && make clean && make -j4 && make check -j4
-  - ./configure --enable-runtime=libidn --enable-builtin=libidn && make clean && make -j4 && make check -j4
-  - ./configure --enable-runtime=libidn --disable-builtin && make clean && make -j4 && make check -j4
-  - ./configure --disable-runtime --enable-builtin=libicu && make clean && make -j4 && make check -j4
-  - ./configure --disable-runtime --enable-builtin=libidn2 && make clean && make -j4 && make check -j4
-  - ./configure --disable-runtime --enable-builtin=libidn && make clean && make -j4 && make check -j4
-  - ./configure --disable-runtime --disable-builtin && make clean && make -j4 && make check -j4
+  - ./configure --enable-runtime=$RUNTIME --enable-builtin=libicu && make clean && make -j4 && make check -j4
+  - ./configure --enable-runtime=$RUNTIME --enable-builtin=libidn2 && make clean && make -j4 && make check -j4
+  - ./configure --enable-runtime=$RUNTIME --enable-builtin=libidn && make clean && make -j4 && make check -j4
+  - ./configure --enable-runtime=$RUNTIME --disable-builtin && make clean && make -j4 && make check -j4
   - ./configure --enable-gtk-doc && make -j4 && make check -j4
   - make distcheck
-before_install:
-  - sudo apt-get -qq update
-  - sudo apt-get -q install autoconf automake autopoint libtool gtk-doc-tools gettext libidn11 libidn11-dev libidn2-0 libidn2-0-dev libicu48 libicu-dev libunistring0 libunistring-dev

From 01a37515243dade1921766e463b5e7516e2fd438 Mon Sep 17 00:00:00 2001
From: Daniel Kahn Gillmor <dkg@fifthhorseman.net>
Date: Fri, 11 Dec 2015 22:59:15 -0500
Subject: [PATCH 18/59] re-fix psl_builtin_outdated()

---
 src/psl.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/psl.c b/src/psl.c
index c7c39ef..7bb1e4d 100644
--- a/src/psl.c
+++ b/src/psl.c
@@ -1098,7 +1098,7 @@ int psl_builtin_outdated(void)
 {
 	struct stat st;
 
-	if (stat(_psl_filename, &st) == 0 && st.st_mtime < _psl_file_time)
+	if (stat(_psl_filename, &st) == 0 && st.st_mtime > _psl_file_time)
 		return 1;
 
 	return 0;

From e252af877f71e561699abaa955b58f0ed595aed2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20R=C3=BChsen?= <tim.ruehsen@gmx.de>
Date: Tue, 15 Dec 2015 20:46:25 +0100
Subject: [PATCH 19/59] Fix ./configure --disable-builtin

---
 src/psl2c.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/psl2c.c b/src/psl2c.c
index 7d02679..09712e1 100644
--- a/src/psl2c.c
+++ b/src/psl2c.c
@@ -305,7 +305,7 @@ int main(int argc, const char **argv)
 	psl_free(psl);
 #else
 	if ((fpout = fopen(argv[argpos + 1], "w"))) {
-		fprintf(fpout, "static _psl_entry_t suffixes[1];\n");
+		fprintf(fpout, "static const unsigned char kDafsa[1];\n");
 		fprintf(fpout, "static time_t _psl_file_time;\n");
 		fprintf(fpout, "static time_t _psl_compile_time;\n");
 		fprintf(fpout, "static int _psl_nsuffixes = 0;\n");

From cd5a11b7a57e11cca0f3dbe6113d6159b7617087 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20R=C3=BChsen?= <tim.ruehsen@gmx.de>
Date: Tue, 15 Dec 2015 21:21:28 +0100
Subject: [PATCH 20/59] Remove ./configure --disable-runtime

With DAFSA as builtin data, we need runtime conversions.
---
 .travis.yml  | 1 -
 configure.ac | 3 ---
 2 files changed, 4 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 3a0823c..4290894 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -10,7 +10,6 @@ env:
     - RUNTIME=libicu
     - RUNTIME=libidn2
     - RUNTIME=libidn
-    - RUNTIME=no
 
 addons:
     apt:
diff --git a/configure.ac b/configure.ac
index 1a8698b..44e70a8 100644
--- a/configure.ac
+++ b/configure.ac
@@ -96,7 +96,6 @@ AC_ARG_ENABLE(runtime,
         libicu [[default]]: IDNA2008 UTS#46 library
         libidn2:          IDNA2008 library (also needs libunistring)
         libidn:           IDNA2003 library (also needs libunistring)
-  --disable-runtime       Do not link runtime IDNA functionality
   ], [
     if test "$enableval" = "libicu" -o "$enableval" = "yes"; then
       enable_runtime=libicu
@@ -107,8 +106,6 @@ AC_ARG_ENABLE(runtime,
     elif test "$enableval" = "libidn"; then
       enable_runtime=libidn
       AC_DEFINE([WITH_LIBIDN], [1], [generate PSL data using libidn])
-    elif test "$enableval" = "no"; then
-      enable_runtime=no
     else
       AC_MSG_ERROR([Unknown value $enableval for --enable-runtime])
     fi

From 620e62348a9bcbddb7e8d282a22f06edb5923db3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20R=C3=BChsen?= <tim.ruehsen@gmx.de>
Date: Fri, 25 Dec 2015 19:40:24 +0100
Subject: [PATCH 21/59] Revert "Remove ./configure --disable-runtime"

This reverts commit cd5a11b7a57e11cca0f3dbe6113d6159b7617087.
---
 .travis.yml  | 1 +
 configure.ac | 3 +++
 2 files changed, 4 insertions(+)

diff --git a/.travis.yml b/.travis.yml
index 4290894..3a0823c 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -10,6 +10,7 @@ env:
     - RUNTIME=libicu
     - RUNTIME=libidn2
     - RUNTIME=libidn
+    - RUNTIME=no
 
 addons:
     apt:
diff --git a/configure.ac b/configure.ac
index 44e70a8..1a8698b 100644
--- a/configure.ac
+++ b/configure.ac
@@ -96,6 +96,7 @@ AC_ARG_ENABLE(runtime,
         libicu [[default]]: IDNA2008 UTS#46 library
         libidn2:          IDNA2008 library (also needs libunistring)
         libidn:           IDNA2003 library (also needs libunistring)
+  --disable-runtime       Do not link runtime IDNA functionality
   ], [
     if test "$enableval" = "libicu" -o "$enableval" = "yes"; then
       enable_runtime=libicu
@@ -106,6 +107,8 @@ AC_ARG_ENABLE(runtime,
     elif test "$enableval" = "libidn"; then
       enable_runtime=libidn
       AC_DEFINE([WITH_LIBIDN], [1], [generate PSL data using libidn])
+    elif test "$enableval" = "no"; then
+      enable_runtime=no
     else
       AC_MSG_ERROR([Unknown value $enableval for --enable-runtime])
     fi

From 093d5eac3d6a2bc04e4c8b67b1f6076ce4aa15d5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20R=C3=BChsen?= <tim.ruehsen@gmx.de>
Date: Sat, 26 Dec 2015 14:15:08 +0100
Subject: [PATCH 22/59] Fix ./configure --disable-runtime

Added runtime punycode generation code from
  http://www.nicemice.net/idn/punycode-spec.gz
---
 contrib/check-hard |   5 +-
 src/psl.c          | 292 ++++++++++++++++++++++++++++++++++++++++++++-
 src/psl2c.c        |   6 +-
 3 files changed, 297 insertions(+), 6 deletions(-)

diff --git a/contrib/check-hard b/contrib/check-hard
index a307ff3..a3a3c3a 100755
--- a/contrib/check-hard
+++ b/contrib/check-hard
@@ -14,7 +14,7 @@ make distclean > /dev/null || true
 
 # We define _GNU_SOURCE to avoid warnings with missing prototypes.
 # C89 does not know snprintf, strdup, strndup, popen, pclose
-CFLAGS="-std=c89 -pedantic -O2 -g -Wall -Wextra -Wstrict-prototypes -Wold-style-definition -Wwrite-strings -Wshadow -Wformat -Wformat-security -Wunreachable-code -Wstrict-prototypes -Wmissing-prototypes -Wold-style-definition -D_GNU_SOURCE"
+CFLAGS="-std=gnu89 -pedantic -O2 -g -Wall -Wextra -Wstrict-prototypes -Wold-style-definition -Wwrite-strings -Wshadow -Wformat -Wformat-security -Wunreachable-code -Wstrict-prototypes -Wmissing-prototypes -Wold-style-definition"
 
 CACHEFILE=$PWD/config_check.cache
 
@@ -40,7 +40,8 @@ for CC in gcc clang; do
   for options in \
   "--enable-runtime=libicu --enable-builtin=libicu" \
   "--enable-runtime=libidn2 --enable-builtin=libidn2" \
-  "--enable-runtime=libidn --enable-builtin=libidn"; do
+  "--enable-runtime=libidn --enable-builtin=libidn" \
+  "--disable-runtime --enable-builtin=libicu"; do
     export DISTCHECK_CONFIGURE_FLAGS="-C --cache-file=$CACHEFILE $options"
     echo
     echo "  *** ./configure $DISTCHECK_CONFIGURE_FLAGS"
diff --git a/src/psl.c b/src/psl.c
index 7bb1e4d..7d434e7 100644
--- a/src/psl.c
+++ b/src/psl.c
@@ -74,6 +74,7 @@
 #include <string.h>
 #include <ctype.h>
 #include <errno.h>
+#include <limits.h> /* for UINT_MAX */
 #include <langinfo.h>
 #include <arpa/inet.h>
 #ifdef HAVE_ALLOCA_H
@@ -325,6 +326,285 @@ static int _suffix_init(_psl_entry_t *suffix, const char *rule, size_t length)
 	return 0;
 }
 
+#if !defined(WITH_LIBIDN) && !defined(WITH_LIBIDN2) && !defined(WITH_LIBICU)
+/*
+ * When configured without runtime IDNA support (./configure --disable-runtime), we need a pure ASCII
+ * representation of non-ASCII characters in labels as found in UTF-8 domain names.
+ * This is because the current DAFSA format used may only hold character values [21..127].
+ *
+  Code copied from http://www.nicemice.net/idn/punycode-spec.gz on
+  2011-01-04 with SHA-1 a966a8017f6be579d74a50a226accc7607c40133
+  labeled punycode-spec 1.0.3 (2006-Mar-24-Thu).  It is modified for
+  libpsl by Tim Rühsen.  License on the original code:
+
+  punycode-spec 1.0.3 (2006-Mar-23-Thu)
+  http://www.nicemice.net/idn/
+  Adam M. Costello
+  http://www.nicemice.net/amc/
+
+  B. Disclaimer and license
+
+    Regarding this entire document or any portion of it (including
+    the pseudocode and C code), the author makes no guarantees and
+    is not responsible for any damage resulting from its use.  The
+    author grants irrevocable permission to anyone to use, modify,
+    and distribute it in any way that does not diminish the rights
+    of anyone else to use, modify, and distribute it, provided that
+    redistributed derivative works do not contain misleading author or
+    version information.  Derivative works need not be licensed under
+    similar terms.
+
+  C. Punycode sample implementation
+
+  punycode-sample.c 2.0.0 (2004-Mar-21-Sun)
+  http://www.nicemice.net/idn/
+  Adam M. Costello
+  http://www.nicemice.net/amc/
+
+  This is ANSI C code (C89) implementing Punycode 1.0.x.
+ */
+enum punycode_status {
+	punycode_success = 0,
+	punycode_bad_input = 1, /* Input is invalid.                       */
+	punycode_big_output = 2, /* Output would exceed the space provided. */
+	punycode_overflow = 3 /* Wider integers needed to process input. */
+};
+
+#ifdef PUNYCODE_UINT
+	typedef PUNYCODE_UINT punycode_uint;
+#elif UINT_MAX >= (1 << 26) - 1
+	typedef unsigned int punycode_uint;
+#else
+	typedef unsigned long punycode_uint;
+#endif
+
+/*** Bootstring parameters for Punycode ***/
+enum {
+	base = 36, tmin = 1, tmax = 26, skew = 38, damp = 700,
+	initial_bias = 72, initial_n = 0x80, delimiter = 0x2D
+};
+
+static char encode_digit(punycode_uint d)
+{
+	return d + 22 + 75 * (d < 26);
+	/*  0..25 map to ASCII a..z or A..Z */
+	/* 26..35 map to ASCII 0..9         */
+}
+#define flagged(bcp) ((punycode_uint)(bcp) - 65 < 26)
+static const punycode_uint maxint = -1;
+
+static punycode_uint adapt(punycode_uint delta, punycode_uint numpoints, int firsttime)
+{
+	punycode_uint k;
+
+	delta = firsttime ? delta / damp : delta >> 1;
+	/* delta >> 1 is a faster way of doing delta / 2 */
+	delta += delta / numpoints;
+
+	for (k = 0; delta > ((base - tmin) * tmax) / 2; k += base) {
+		delta /= base - tmin;
+	}
+
+	return k + (base - tmin + 1) * delta / (delta + skew);
+}
+
+static enum punycode_status punycode_encode(
+	size_t input_length_orig,
+	const punycode_uint input[],
+	size_t *output_length,
+	char output[])
+{
+	punycode_uint input_length, n, delta, h, b, bias, j, m, q, k, t;
+	size_t out, max_out;
+
+	/* The Punycode spec assumes that the input length is the same type */
+	/* of integer as a code point, so we need to convert the size_t to  */
+	/* a punycode_uint, which could overflow.                           */
+
+	if (input_length_orig > maxint)
+		return punycode_overflow;
+
+	input_length = (punycode_uint) input_length_orig;
+
+	/* Initialize the state: */
+
+	n = initial_n;
+	delta = 0;
+	out = 0;
+	max_out = *output_length;
+	bias = initial_bias;
+
+	/* Handle the basic code points: */
+	for (j = 0; j < input_length; ++j) {
+		if (input[j] < 0x80) {
+			if (max_out - out < 2)
+				return punycode_big_output;
+			output[out++] = (char) input[j];
+		}
+		/* else if (input[j] < n) return punycode_bad_input; */
+		/* (not needed for Punycode with unsigned code points) */
+	}
+
+	h = b = (punycode_uint) out;
+	/* cannot overflow because out <= input_length <= maxint */
+
+	/* h is the number of code points that have been handled, b is the  */
+	/* number of basic code points, and out is the number of ASCII code */
+	/* points that have been output.                                    */
+
+	if (b > 0)
+		output[out++] = delimiter;
+
+	/* Main encoding loop: */
+
+	while (h < input_length) {
+		/* All non-basic code points < n have been     */
+		/* handled already.  Find the next larger one: */
+
+		for (m = maxint, j = 0; j < input_length; ++j) {
+			/* if (basic(input[j])) continue; */
+			/* (not needed for Punycode) */
+			if (input[j] >= n && input[j] < m)
+				m = input[j];
+		}
+
+		/* Increase delta enough to advance the decoder's    */
+		/* <n,i> state to <m,0>, but guard against overflow: */
+
+		if (m - n > (maxint - delta) / (h + 1))
+			return punycode_overflow;
+		delta += (m - n) * (h + 1);
+		n = m;
+
+		for (j = 0; j < input_length; ++j) {
+			/* Punycode does not need to check whether input[j] is basic: */
+			if (input[j] < n /* || basic(input[j]) */) {
+				if (++delta == 0)
+					return punycode_overflow;
+			}
+
+			if (input[j] == n) {
+				/* Represent delta as a generalized variable-length integer: */
+
+				for (q = delta, k = base;; k += base) {
+					if (out >= max_out)
+						return punycode_big_output;
+					t = k <= bias /* + tmin */ ? tmin : /* +tmin not needed */
+						k >= bias + tmax ? tmax : k - bias;
+					if (q < t)
+						break;
+					output[out++] = encode_digit(t + (q - t) % (base - t));
+					q = (q - t) / (base - t);
+				}
+
+				output[out++] = encode_digit(q);
+				bias = adapt(delta, h + 1, h == b);
+				delta = 0;
+				++h;
+			}
+		}
+
+		++delta, ++n;
+	}
+
+	*output_length = out;
+	return punycode_success;
+}
+
+static ssize_t _utf8_to_utf32(const char *in, size_t inlen, punycode_uint *out, size_t outlen)
+{
+	size_t n = 0;
+	unsigned char *s;
+
+	if (!outlen)
+		return -1;
+
+	outlen--;
+
+	s = alloca(inlen + 1);
+	memcpy(s, in, inlen);
+	s[inlen] = 0;
+
+	while (*s && n < outlen) {
+		if ((*s & 0x80) == 0) { /* 0xxxxxxx ASCII char */
+			out[n++] = *s;
+			s++;
+		} else if ((*s & 0xE0) == 0xC0) /* 110xxxxx 10xxxxxx */ {
+			if ((s[1] & 0xC0) != 0x80)
+				return -1;
+			out[n++] = ((*s & 0x1F) << 6) | (s[1] & 0x3F);
+			s += 2;
+		} else if ((*s & 0xF0) == 0xE0) /* 1110xxxx 10xxxxxx 10xxxxxx */ {
+			if ((s[1] & 0xC0) != 0x80 || (s[2] & 0xC0) != 0x80)
+				return -1;
+			out[n++] = ((*s & 0x0F) << 12) | ((s[1] & 0x3F) << 6) | (s[2] & 0x3F);
+			s += 3;
+		} else if ((*s & 0xF8) == 0xF0) /* 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx */ {
+			if ((s[1] & 0xC0) != 0x80 || (s[2] & 0xC0) != 0x80 || (s[3] & 0xC0) != 0x80)
+				return -1;
+			out[n++] = ((*s & 0x07) << 18) | ((s[1] & 0x3F) << 12) | ((s[1] & 0x3F) << 6) | (s[2] & 0x3F);
+			s += 4;
+		} else
+			return -1;
+	}
+
+	return n;
+}
+
+static int _mem_is_ascii(const char *s, size_t n)
+{
+	while (n--)
+		if (*((unsigned char *)s++) >= 128)
+			return 0;
+
+	return 1;
+}
+
+static int _domain_to_punycode(const char *domain, char *out, size_t outsize)
+{
+	size_t outlen = 0, labellen;
+	punycode_uint input[256];
+	const char *label, *e;
+
+	for (e = label = domain; e; label = e + 1) {
+		e = strchr(label, '.');
+		labellen = e ? (size_t) (e - label) : strlen(label);
+		/* printf("s=%s inlen=%zd\n", label, labellen); */
+
+		if (_mem_is_ascii(label, labellen)) {
+			if (outlen + labellen + (e != NULL)>= outsize)
+				return 1;
+
+			/* printf("outlen=%zd labellen=%zd\n", outlen, labellen); */
+			memcpy(out + outlen, label, labellen);
+			outlen += labellen;
+		} else {
+			ssize_t inputlen = 0;
+
+			if (outlen + labellen + (e != NULL) + 4 >= outsize)
+				return 1;
+
+			if ((inputlen = _utf8_to_utf32(label, labellen, input, sizeof (input) / sizeof (input[0]))) < 0)
+				return 1;
+
+			memcpy(out + outlen, "xn--", 4);
+			outlen += 4;
+
+			labellen = outsize - outlen;
+			/* printf("n=%zd space_left=%zd\n", n, labellen); */
+			if (punycode_encode(inputlen, input, &labellen, out + outlen))
+				return 1;
+			outlen += labellen;
+		}
+
+		if (e)
+			out[outlen++] = '.';
+		out[outlen] = 0;
+	}
+
+	return 0;
+}
+#endif
 
 static inline int _isspace_ascii(const char c)
 {
@@ -466,6 +746,14 @@ static int _psl_idna_toASCII(_psl_idna_t *idna _UNUSED, const char *utf8, char *
 		ret = 0;
 	} /* else
 		fprintf(_(stderr, "toASCII failed (%d): %s\n"), rc, idna_strerror(rc)); */
+#else
+	char lookupname[128];
+
+	if (_domain_to_punycode(utf8, lookupname, sizeof(lookupname)) == 0) {
+		if (ascii)
+			*ascii = strdup(lookupname);
+		ret = 0;
+	}
 #endif
 
 	return ret;
@@ -541,6 +829,8 @@ static int _psl_is_public_suffix(const psl_ctx_t *psl, const char *domain, int t
 		suffix.length = p - suffix.label;
 	}
 
+	printf("domain=%s label=%s\n", domain, suffix.label);
+
 	if (psl == &_builtin_psl) {
 		int rc = LookupStringInFixedSet(kDafsa, sizeof(kDafsa), suffix.label, suffix.length);
 		if (rc != -1) {
@@ -1250,7 +1540,7 @@ int psl_is_cookie_domain_acceptable(const psl_ctx_t *psl, const char *hostname,
  *
  * Since: 0.4
  */
-psl_error_t psl_str_to_utf8lower(const char *str, const char *encoding, const char *locale _UNUSED, char **lower)
+psl_error_t psl_str_to_utf8lower(const char *str, const char *encoding _UNUSED, const char *locale _UNUSED, char **lower)
 {
 	int ret = PSL_ERR_INVALID_ARG;
 
diff --git a/src/psl2c.c b/src/psl2c.c
index 09712e1..f8255e1 100644
--- a/src/psl2c.c
+++ b/src/psl2c.c
@@ -143,11 +143,11 @@ static void _print_psl_entries_dafsa(FILE *fpout, const _psl_vector_t *v)
 		fprintf(fpout, "/* automatically generated by psl2c (punycode generated with libicu/%s) */\n", version);
 	} while (0);
 #elif defined(BUILTIN_GENERATOR_LIBIDN2)
-		fprintf(fpout, "/* automatically generated by psl2c (punycode generated with libidn2/%s) */\n", idn2_check_version(NULL));
+	fprintf(fpout, "/* automatically generated by psl2c (punycode generated with libidn2/%s) */\n", idn2_check_version(NULL));
 #elif defined(BUILTIN_GENERATOR_LIBIDN)
-		fprintf(fpout, "/* automatically generated by psl2c (punycode generated with libidn/%s) */\n", stringprep_check_version(NULL));
+	fprintf(fpout, "/* automatically generated by psl2c (punycode generated with libidn/%s) */\n", stringprep_check_version(NULL));
 #else
-	fprintf(fpout, "/* automatically generated by psl2c (without punycode support) */\n");
+	fprintf(fpout, "/* automatically generated by psl2c (punycode generated internally) */\n");
 #endif
 
 	if ((fp = fopen("in.tmp", "w"))) {

From 5363290cbe50113826854c7d800da621b00d786e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20R=C3=BChsen?= <tim.ruehsen@gmx.de>
Date: Sat, 26 Dec 2015 14:29:10 +0100
Subject: [PATCH 23/59] Remove debugging printf

---
 src/psl.c | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/psl.c b/src/psl.c
index 7d434e7..c320ed1 100644
--- a/src/psl.c
+++ b/src/psl.c
@@ -829,8 +829,6 @@ static int _psl_is_public_suffix(const psl_ctx_t *psl, const char *domain, int t
 		suffix.length = p - suffix.label;
 	}
 
-	printf("domain=%s label=%s\n", domain, suffix.label);
-
 	if (psl == &_builtin_psl) {
 		int rc = LookupStringInFixedSet(kDafsa, sizeof(kDafsa), suffix.label, suffix.length);
 		if (rc != -1) {

From e63ff8abfc9fbed7eaf6d253b82a25bc8d21743b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20R=C3=BChsen?= <tim.ruehsen@gmx.de>
Date: Sat, 26 Dec 2015 14:44:57 +0100
Subject: [PATCH 24/59] Avoid libicu dependency with --enable-runtime=no

---
 configure.ac | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/configure.ac b/configure.ac
index 1a8698b..9d29105 100644
--- a/configure.ac
+++ b/configure.ac
@@ -154,8 +154,10 @@ if test "$enable_runtime" = "libicu" -o "$enable_builtin" = "libicu"; then
   # using AC_SEARCH_LIBS also don't work since functions have the library version appended
   PKG_CHECK_MODULES([LIBICU], [icu-uc], [
     HAVE_LIBICU=yes
-    LIBS="$LIBICU_LIBS $LIBS"
-    CFLAGS="$LIBICU_CFLAGS $CFLAGS"
+    if test "$enable_runtime" = "libicu"; then
+      LIBS="$LIBICU_LIBS $LIBS"
+      CFLAGS="$LIBICU_CFLAGS $CFLAGS"
+    fi
   ], [
     OLDLIBS=$LIBS
     LIBS="-licuuc $LIBS"

From 82e9445493c13788db28c794b3c1654ff216121e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20R=C3=BChsen?= <tim.ruehsen@gmx.de>
Date: Tue, 29 Dec 2015 16:53:47 +0100
Subject: [PATCH 25/59] Add psl2c --binary to create DAFSA binary file from PSL

---
 src/make_dafsa.py |  37 ++++++++++++----
 src/psl2c.c       | 105 ++++++++++++++++++++++++----------------------
 2 files changed, 84 insertions(+), 58 deletions(-)

diff --git a/src/make_dafsa.py b/src/make_dafsa.py
index 8268271..c3766ec 100755
--- a/src/make_dafsa.py
+++ b/src/make_dafsa.py
@@ -430,12 +430,22 @@ def to_cxx(data):
   return text
 
 
-def words_to_cxx(words):
+def words_to_whatever(words, converter):
   """Generates C++ code from a word list"""
   dafsa = to_dafsa(words)
   for fun in (reverse, join_suffixes, reverse, join_suffixes, join_labels):
     dafsa = fun(dafsa)
-  return to_cxx(encode(dafsa))
+  return converter(encode(dafsa))
+
+
+def words_to_cxx(words):
+  """Generates C++ code from a word list"""
+  return words_to_whatever(words, to_cxx)
+
+
+def words_to_binary(words):
+  """Generates C++ code from a word list"""
+  return words_to_whatever(words, bytearray)
 
 
 def parse_gperf(infile):
@@ -457,15 +467,24 @@ def parse_gperf(infile):
 
 
 def main():
-  if len(sys.argv) != 3:
-    print('usage: %s infile outfile' % sys.argv[0])
+  if len(sys.argv) < 3:
+    print('usage: %s [--binary] infile outfile' % sys.argv[0])
     return 1
-  if sys.argv[1] == '-':
-    with open(sys.argv[2], 'w') as outfile:
-      outfile.write(words_to_cxx(parse_gperf(sys.stdin)))
+
+  argpos = 1
+  converter = words_to_cxx
+
+  if sys.argv[argpos] == '--binary':
+    converter = words_to_binary
+    argpos += 1
+
+  if sys.argv[argpos] == '-':
+    with open(sys.argv[argpos + 1], 'w') as outfile:
+      outfile.write(converter(parse_gperf(sys.stdin)))
   else:
-    with open(sys.argv[1], 'r') as infile, open(sys.argv[2], 'w') as outfile:
-      outfile.write(words_to_cxx(parse_gperf(infile)))
+    with open(sys.argv[argpos], 'r') as infile, open(sys.argv[argpos + 1], 'w') as outfile:
+      outfile.write(converter(parse_gperf(infile)))
+
   return 0
 
 
diff --git a/src/psl2c.c b/src/psl2c.c
index f8255e1..47db065 100644
--- a/src/psl2c.c
+++ b/src/psl2c.c
@@ -180,52 +180,50 @@ static void _print_psl_entries_dafsa(FILE *fpout, const _psl_vector_t *v)
 	unlink("in.tmp");
 	unlink("out.tmp");
 }
+#endif /* _GENERATE_BUILTIN_DATA */
 
-#if 0
-#if !defined(WITH_LIBICU) && !defined(WITH_IDN2)
-static int _str_needs_encoding(const char *s)
+static int _print_psl_entries_dafsa_binary(const char *fname, const _psl_vector_t *v)
 {
-	while (*s && *((unsigned char *)s) < 128) s++;
+	FILE *fp;
+	int ret = 0, it, rc;
+	char cmd[256];
 
-	return !!*s;
-}
+	if ((fp = fopen("in.tmp", "w"))) {
+		for (it = 0; it < v->cur; it++) {
+			_psl_entry_t *e = _vector_get(v, it);
+			unsigned char *s = (unsigned char *)e->label_buf;
 
-static void _add_punycode_if_needed(_psl_vector_t *v)
-{
-	int it, n;
+			/* search for non-ASCII label and skip it */
+			while (*s && *s < 128) s++;
+			if (*s) continue;
 
-	/* do not use 'it < v->cur' since v->cur is changed by _vector_add() ! */
-	for (it = 0, n = v->cur; it < n; it++) {
-		_psl_entry_t *e = _vector_get(v, it);
-
-		if (_str_needs_encoding(e->label_buf)) {
-			_psl_entry_t suffix, *suffixp;
-			char lookupname[64] = "";
-
-			/* this is much slower than the libidn2 API but should have no license issues */
-			FILE *pp;
-			char cmd[16 + sizeof(e->label_buf)];
-			snprintf(cmd, sizeof(cmd), "idn2 '%s'", e->label_buf);
-			if ((pp = popen(cmd, "r"))) {
-				if (fscanf(pp, "%63s", lookupname) >= 1 && strcmp(e->label_buf, lookupname)) {
-					/* fprintf(stderr, "idn2 '%s' -> '%s'\n", e->label_buf, lookupname); */
-					_suffix_init(&suffix, lookupname, strlen(lookupname));
-					suffix.wildcard = e->wildcard;
-					suffixp = _vector_get(v, _vector_add(v, &suffix));
-					suffixp->label = suffixp->label_buf; /* set label to changed address */
-				}
-				pclose(pp);
-			} else
-				fprintf(stderr, "Failed to call popen(%s, \"r\")\n", cmd);
+			fprintf(fp, "%s, %X\n", e->label_buf, (int) (e->flags & 0x0F));
 		}
+
+		fclose(fp);
+	} else {
+		fprintf(stderr, "Failed to write open 'in.tmp'\n");
+		return 3;
 	}
 
-	_vector_sort(v);
-}
-#endif /* !defined(WITH_LIBICU) && !defined(WITH_IDN2) */
-#endif
+	snprintf(cmd, sizeof(cmd), MAKE_DAFSA " --binary in.tmp %s", fname);
+	if ((rc = system(cmd))) {
+		fprintf(stderr, "Failed to execute '%s' (%d)\n", cmd, rc);
+		ret = 2;
+	}
 
-#endif /* _GENERATE_BUILTIN_DATA */
+	unlink("in.tmp");
+	return ret;
+}
+
+static void usage(void)
+{
+	fprintf(stderr, "Usage: psl2c [--binary] <infile> <outfile>\n");
+	fprintf(stderr, "  <infile>  is the 'public_suffix_list.dat', lowercase UTF-8 encoded\n");
+	fprintf(stderr, "  <outfile> is the the filename to be generated from <infile>\n");
+	fprintf(stderr, "  --binary  Generate binary DAFSA output (default: C code for psl.c)\n");
+	exit(1);
+}
 
 int main(int argc, const char **argv)
 {
@@ -233,20 +231,34 @@ int main(int argc, const char **argv)
 #ifdef _GENERATE_BUILTIN_DATA
 	psl_ctx_t *psl;
 #endif
-	int ret = 0, argpos = 1;
+	int ret = 0, argpos = 1, binary = 0;
 
-	if (argc - argpos != 2) {
-		fprintf(stderr, "Usage: psl2c <infile> <outfile>\n");
-		fprintf(stderr, "  <infile>  is the 'public_suffix_list.dat', lowercase UTF-8 encoded\n");
-		fprintf(stderr, "  <outfile> is the the C filename to be generated from <infile>\n");
-		return 1;
+	if (argc < 3)
+		usage();
+
+	if (strcmp(argv[argpos], "--binary") == 0) {
+		argpos++;
+		binary = 1;
+	}
+
+	if (argc - argpos != 2)
+		usage();
+
+	if (binary) {
+		if (!(psl = psl_load_file(argv[argpos])))
+			return 2;
+
+		ret = _print_psl_entries_dafsa_binary(argv[argpos + 1], psl->suffixes);
+
+		psl_free(psl);
+		return ret;
 	}
 
 #ifdef _GENERATE_BUILTIN_DATA
 	if (!(psl = psl_load_file(argv[argpos])))
 		return 2;
 
-	/* look for ambigious or double entries */
+	/* look for ambiguous or double entries */
 /*	if (_check_psl(psl)) {
 		psl_free(psl);
 		return 5;
@@ -260,11 +272,6 @@ int main(int argc, const char **argv)
 		char *abs_srcfile;
 		const char *source_date_epoch = NULL;
 
-#if 0
-		/* include library code did not generate punycode, so let's do it for the builtin data */
-		_add_punycode_if_needed(psl->suffixes);
-#endif
-
 		_print_psl_entries_dafsa(fpout, psl->suffixes);
 
 		snprintf(cmd, cmdsize, "sha1sum %s", argv[argpos]);

From 355edc152faca3e72cb63a23f52ab3a98953643e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20R=C3=BChsen?= <tim.ruehsen@gmx.de>
Date: Tue, 29 Dec 2015 17:20:28 +0100
Subject: [PATCH 26/59] Fix for previous commit

---
 src/psl2c.c | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/src/psl2c.c b/src/psl2c.c
index 47db065..c4c280f 100644
--- a/src/psl2c.c
+++ b/src/psl2c.c
@@ -45,8 +45,6 @@
 #	define _GENERATE_BUILTIN_DATA
 #endif
 
-#ifdef _GENERATE_BUILTIN_DATA
-
 #include <libpsl.h>
 
 /* here we include the library source code to have access to internal functions and data structures */
@@ -54,6 +52,8 @@
 #	include "psl.c"
 #undef _LIBPSL_INCLUDED_BY_PSL2C
 
+#ifdef _GENERATE_BUILTIN_DATA
+
 #if 0
 static int _check_psl(const psl_ctx_t *psl)
 {
@@ -228,9 +228,7 @@ static void usage(void)
 int main(int argc, const char **argv)
 {
 	FILE *fpout;
-#ifdef _GENERATE_BUILTIN_DATA
 	psl_ctx_t *psl;
-#endif
 	int ret = 0, argpos = 1, binary = 0;
 
 	if (argc < 3)

From c699e3c441830407ec118e3b6d9c003e9ce40957 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20R=C3=BChsen?= <tim.ruehsen@gmx.de>
Date: Wed, 30 Dec 2015 17:52:48 +0100
Subject: [PATCH 27/59] Add --input-format and --output-format to make_dafsa.py

---
 src/make_dafsa.py | 144 ++++++++++++++++++++++++++++++++++++++--------
 src/psl.c         |   2 +-
 2 files changed, 121 insertions(+), 25 deletions(-)

diff --git a/src/make_dafsa.py b/src/make_dafsa.py
index c3766ec..0456191 100755
--- a/src/make_dafsa.py
+++ b/src/make_dafsa.py
@@ -206,14 +206,14 @@ def to_dafsa(words):
   """
   if not words:
     raise InputError('The domain list must not be empty')
-  def ToNodes(word):
+  def to_nodes(word):
     """Split words into characters"""
     if not 0x1F < ord(word[0]) < 0x80:
       raise InputError('Domain names must be printable 7-bit ASCII')
     if len(word) == 1:
       return chr(ord(word[0]) & 0x0F), [None]
-    return word[0], [ToNodes(word[1:])]
-  return [ToNodes(word) for word in words]
+    return word[0], [to_nodes(word[1:])]
+  return [to_nodes(word) for word in words]
 
 
 def to_words(node):
@@ -254,8 +254,8 @@ def join_labels(dafsa):
   """Generates a new DAFSA where internal nodes are merged if there is a one to
   one connection.
   """
-  parentcount = { id(None): 2 }
-  nodemap = { id(None): None }
+  parentcount = {id(None): 2}
+  nodemap = {id(None): None}
 
   def count_parents(node):
     """Count incoming references"""
@@ -286,7 +286,7 @@ def join_suffixes(dafsa):
   """Generates a new DAFSA where nodes that represent the same word lists
   towards the sink are merged.
   """
-  nodemap = { frozenset(('',)): None }
+  nodemap = {frozenset(('',)): None}
 
   def join(node):
     """Returns a macthing node. A new node is created if no matching node
@@ -343,7 +343,7 @@ def encode_links(children, offsets, current):
     return []
   guess = 3 * len(children)
   assert children
-  children = sorted(children, key = lambda x: -offsets[id(x)])
+  children = sorted(children, key=lambda x: -offsets[id(x)])
   while True:
     offset = current + guess
     buf = []
@@ -448,13 +448,10 @@ def words_to_binary(words):
   return words_to_whatever(words, bytearray)
 
 
-def parse_gperf(infile):
-  """Parses gperf file and extract strings and return code"""
+def parse_psl2c(infile):
+  """Parses file generated by psl2c and extract strings and return code"""
   lines = [line.strip() for line in infile]
-  # Extract strings after the first '%%' and before the second '%%'.
-  #begin = lines.index('%%') + 1
-  #end = lines.index('%%', begin)
-  #lines = lines[begin:end]
+
   for line in lines:
     if line[-3:-1] != ', ':
       raise InputError('Expected "domainname, <digit>", found "%s"' % line)
@@ -463,27 +460,126 @@ def parse_gperf(infile):
     if line[-1] not in '0123456789ABCDEF':
       raise InputError('Expected value to be one of {0,1,2,4,5}, found "%s"' %
                        line[-1])
+
+#  with open("gperf.out", 'w') as outfile:
+#    for line in lines:
+#      outfile.write(line[:-3] + line[-1] + "\n")
+
   return [line[:-3] + line[-1] for line in lines]
 
 
+def parse_psl(infile):
+  """Parses PSL file and extract strings and return code"""
+  PSL_FLAG_EXCEPTION = (1<<0)
+  PSL_FLAG_WILDCARD = (1<<1)
+  PSL_FLAG_ICANN = (1<<2) # entry of ICANN section
+  PSL_FLAG_PRIVATE = (1<<3) # entry of PRIVATE section
+  PSL_FLAG_PLAIN = (1<<4) #just used for PSL syntax checking
+
+  psl = {}
+  section = 0
+
+  for line in infile:
+    line = line.strip()
+    if not line:
+      continue
+
+    if line.startswith("//"):
+      if section == 0:
+        if "===BEGIN ICANN DOMAINS===" in line:
+          section = PSL_FLAG_ICANN
+        elif section == 0 and "===BEGIN PRIVATE DOMAINS===" in line:
+          section = PSL_FLAG_PRIVATE
+      elif section == PSL_FLAG_ICANN and "===END ICANN DOMAINS===" in line:
+        section = 0
+      elif section == PSL_FLAG_PRIVATE and "===END PRIVATE DOMAINS===" in line:
+        section = 0
+      continue # skip comments
+
+    if line[0] == '!':
+      flags = PSL_FLAG_EXCEPTION | section
+      line = line[1:]
+    elif line[0] == '*':
+      if line[1] != '.':
+        print 'Unsupported kind of rule (ignored): %s' % line
+        continue
+      flags = PSL_FLAG_WILDCARD | PSL_FLAG_PLAIN | section
+      line = line[2:]
+    else:
+      if not '.' in line:
+        continue # we do not need an explicit plain TLD rule, already covered by implicit '*' rule
+      flags = PSL_FLAG_PLAIN | section
+
+    line = line.decode('utf-8').encode("idna")
+
+    if line in psl:
+      """Found existing entry:
+         Combination of exception and plain rule is ambiguous
+           !foo.bar
+            foo.bar
+
+         Allowed:
+           !foo.bar + *.foo.bar
+            foo.bar + *.foo.bar
+      """
+      print('Found %s/%X (now %X)' % line, psl[line], flags)
+      continue
+
+    psl[line] = flags
+
+#  with open("psl.out", 'w') as outfile:
+#    for (domain, flags) in psl.iteritems():
+#      outfile.write(domain + "%X" % (flags & 0x0F) + "\n")
+
+  return [domain + "%X" % (flags & 0x0F) for (domain, flags) in psl.iteritems()]
+
+
+def usage():
+  """Prints the usage"""
+  print 'usage: %s [options] infile outfile' % sys.argv[0]
+  print '  --input-format=psl2c    infile has been generated by libpsl/psl2c utility (default)'
+  print '  --input-format=psl      infile is a Public Suffix List file'
+  print '  --output-format=cxx     Write DAFSA as C/C++ code'
+  print '  --output-format=binary  Write DAFSA binary data'
+  exit(1)
+
+
 def main():
+  """Convert PSL file into C or binary DAFSA file"""
   if len(sys.argv) < 3:
-    print('usage: %s [--binary] infile outfile' % sys.argv[0])
-    return 1
+    usage()
 
-  argpos = 1
   converter = words_to_cxx
+  parser = parse_psl2c
 
-  if sys.argv[argpos] == '--binary':
-    converter = words_to_binary
-    argpos += 1
+  for arg in sys.argv[1:-2]:
+    if arg.startswith('--input-format='):
+      value = arg[15:].lower()
+      if value == 'psl':
+        parser = parse_psl
+      elif value == 'psl2c':
+        parser = parse_psl2c
+      else:
+        print "Unknown input format '%s'" % value
+        return 1
+    elif arg.startswith('--output-format='):
+      value = arg[16:].lower()
+      if value == 'binary':
+        converter = words_to_binary
+      elif value == 'cxx':
+        converter = words_to_cxx
+      else:
+        print "Unknown output format '%s'" % value
+        return 1
+    else:
+      usage()
 
-  if sys.argv[argpos] == '-':
-    with open(sys.argv[argpos + 1], 'w') as outfile:
-      outfile.write(converter(parse_gperf(sys.stdin)))
+  if sys.argv[-2] == '-':
+    with open(sys.argv[-1], 'w') as outfile:
+      outfile.write(converter(parser(sys.stdin)))
   else:
-    with open(sys.argv[argpos], 'r') as infile, open(sys.argv[argpos + 1], 'w') as outfile:
-      outfile.write(converter(parse_gperf(infile)))
+    with open(sys.argv[-2], 'r') as infile, open(sys.argv[-1], 'w') as outfile:
+      outfile.write(converter(parser(infile)))
 
   return 0
 
diff --git a/src/psl.c b/src/psl.c
index c320ed1..49bfdbd 100644
--- a/src/psl.c
+++ b/src/psl.c
@@ -1170,7 +1170,7 @@ psl_ctx_t *psl_load_fp(FILE *fp)
 
 			if ((index = _vector_find(psl->suffixes, &suffix)) >= 0) {
 				/* Found existing entry:
-				 * Combination of exception and plain rule is ambigous
+				 * Combination of exception and plain rule is ambiguous
 				 * !foo.bar
 				 * foo.bar
 				 *

From cde5e53ea603e21c6dab1162bfc786c9786552e8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20R=C3=BChsen?= <tim.ruehsen@gmx.de>
Date: Fri, 1 Jan 2016 15:44:24 +0100
Subject: [PATCH 28/59] Remove psl_builtin_compile_time() for reproducable
 builds

---
 docs/libpsl/libpsl-sections.txt |  1 -
 include/libpsl.h.in             |  4 ----
 src/psl.c                       | 17 -----------------
 src/psl2c.c                     |  5 -----
 tests/test-is-public-builtin.c  |  3 ---
 tools/psl.c                     |  1 -
 6 files changed, 31 deletions(-)

diff --git a/docs/libpsl/libpsl-sections.txt b/docs/libpsl/libpsl-sections.txt
index 643a778..7dede1f 100644
--- a/docs/libpsl/libpsl-sections.txt
+++ b/docs/libpsl/libpsl-sections.txt
@@ -19,7 +19,6 @@ psl_registrable_domain
 psl_suffix_count
 psl_suffix_exception_count
 psl_suffix_wildcard_count
-psl_builtin_compile_time
 psl_builtin_file_time
 psl_builtin_sha1sum
 psl_builtin_filename
diff --git a/include/libpsl.h.in b/include/libpsl.h.in
index d4c781b..59fa3f6 100644
--- a/include/libpsl.h.in
+++ b/include/libpsl.h.in
@@ -125,10 +125,6 @@ int
 int
 	psl_suffix_wildcard_count(const psl_ctx_t *psl);
 
-/* returns compilation time */
-time_t
-	psl_builtin_compile_time(void);
-
 /* returns mtime of PSL source file */
 time_t
 	psl_builtin_file_time(void);
diff --git a/src/psl.c b/src/psl.c
index 49bfdbd..fb35c2f 100644
--- a/src/psl.c
+++ b/src/psl.c
@@ -185,7 +185,6 @@ struct _psl_ctx_st {
 	/* if this source file is included by psl2c.c, provide empty builtin data */
 	static const unsigned char kDafsa[1];
 	static time_t _psl_file_time;
-	static time_t _psl_compile_time;
 	static int _psl_nsuffixes;
 	static int _psl_nexceptions;
 	static int _psl_nwildcards;
@@ -1304,22 +1303,6 @@ int psl_suffix_wildcard_count(const psl_ctx_t *psl)
 		return 0;
 }
 
-/**
- * psl_builtin_compile_time:
- *
- * This function returns the time when the Publix Suffix List has been compiled into C code (by psl2c).
- *
- * If the generation of built-in data has been disabled during compilation, 0 will be returned.
- *
- * Returns: time_t value or 0.
- *
- * Since: 0.1
- */
-time_t psl_builtin_compile_time(void)
-{
-	return _psl_compile_time;
-}
-
 /**
  * psl_builtin_file_time:
  *
diff --git a/src/psl2c.c b/src/psl2c.c
index c4c280f..0cb8998 100644
--- a/src/psl2c.c
+++ b/src/psl2c.c
@@ -283,10 +283,6 @@ int main(int argc, const char **argv)
 			st.st_mtime = 0;
 		fprintf(fpout, "static time_t _psl_file_time = %lu;\n", st.st_mtime);
 
-		if ((source_date_epoch = getenv("SOURCE_DATE_EPOCH")))
-			fprintf(fpout, "static time_t _psl_compile_time = %lu;\n", atol(source_date_epoch));
-		else
-			fprintf(fpout, "static time_t _psl_compile_time = %lu;\n", time(NULL));
 		fprintf(fpout, "static int _psl_nsuffixes = %d;\n", psl->nsuffixes);
 		fprintf(fpout, "static int _psl_nexceptions = %d;\n", psl->nexceptions);
 		fprintf(fpout, "static int _psl_nwildcards = %d;\n", psl->nwildcards);
@@ -312,7 +308,6 @@ int main(int argc, const char **argv)
 	if ((fpout = fopen(argv[argpos + 1], "w"))) {
 		fprintf(fpout, "static const unsigned char kDafsa[1];\n");
 		fprintf(fpout, "static time_t _psl_file_time;\n");
-		fprintf(fpout, "static time_t _psl_compile_time;\n");
 		fprintf(fpout, "static int _psl_nsuffixes = 0;\n");
 		fprintf(fpout, "static int _psl_nexceptions = 0;\n");
 		fprintf(fpout, "static int _psl_nwildcards = 0;\n");
diff --git a/tests/test-is-public-builtin.c b/tests/test-is-public-builtin.c
index 0aa5762..70603ea 100644
--- a/tests/test-is-public-builtin.c
+++ b/tests/test-is-public-builtin.c
@@ -104,9 +104,6 @@ static void test_psl(void)
 		}
 	}
 
-	printf("psl_builtin_compile_time()=%ld\n", psl_builtin_compile_time());
-	psl_builtin_compile_time() == 0 ? failed++ : ok++;
-
 	printf("psl_builtin_file_time()=%ld\n", psl_builtin_file_time());
 	psl_builtin_file_time() == 0 ? failed++ : ok++;
 
diff --git a/tools/psl.c b/tools/psl.c
index fb3972c..4303a5b 100644
--- a/tools/psl.c
+++ b/tools/psl.c
@@ -211,7 +211,6 @@ int main(int argc, const char *const *argv)
 			printf("builtin exceptions: %d\n", psl_suffix_exception_count(psl));
 			printf("builtin wildcards: %d\n", psl_suffix_wildcard_count(psl));
 			printf("builtin filename: %s\n", psl_builtin_filename());
-			printf("builtin compile time: %ld (%s)\n", psl_builtin_compile_time(), time2str(psl_builtin_compile_time()));
 			printf("builtin file time: %ld (%s)\n", psl_builtin_file_time(), time2str(psl_builtin_file_time()));
 			printf("builtin SHA1 file hash: %s\n", psl_builtin_sha1sum());
 			printf("builtin outdated: %d\n", psl_builtin_outdated());

From c9d76e4898d0eda32c67305b6f763d4cfa90b3d8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20R=C3=BChsen?= <tim.ruehsen@gmx.de>
Date: Fri, 1 Jan 2016 17:20:30 +0100
Subject: [PATCH 29/59] Remove unused variable source_date_epoch

---
 src/psl2c.c | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/psl2c.c b/src/psl2c.c
index 0cb8998..29fdc4e 100644
--- a/src/psl2c.c
+++ b/src/psl2c.c
@@ -268,7 +268,6 @@ int main(int argc, const char **argv)
 		size_t cmdsize = 16 + strlen(argv[argpos]);
 		char *cmd = alloca(cmdsize), checksum[64] = "";
 		char *abs_srcfile;
-		const char *source_date_epoch = NULL;
 
 		_print_psl_entries_dafsa(fpout, psl->suffixes);
 

From 23345f5f37684832dd672a7a247c653e83141a53 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20R=C3=BChsen?= <tim.ruehsen@gmx.de>
Date: Fri, 1 Jan 2016 22:31:01 +0100
Subject: [PATCH 30/59] Convert lookup_string_in_fixed_set.c into UTF-8

---
 src/lookup_string_in_fixed_set.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/lookup_string_in_fixed_set.c b/src/lookup_string_in_fixed_set.c
index bdec2e5..81a4e4d 100644
--- a/src/lookup_string_in_fixed_set.c
+++ b/src/lookup_string_in_fixed_set.c
@@ -2,7 +2,7 @@
  * Use of this source code is governed by a BSD-style license that can be
  * found in the LICENSE.chromium file.
  *
- * Converted to C89 2015 by Tim Rühsen
+ * Converted to C89 2015 by Tim Rühsen
  */
 
 #include <stddef.h>

From 1604cb3dca9384683000ceaf39219867c1bf367c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20R=C3=BChsen?= <tim.ruehsen@gmx.de>
Date: Fri, 1 Jan 2016 22:32:11 +0100
Subject: [PATCH 31/59] Fix make_dafsa.py to generate 4 bit return values

---
 src/make_dafsa.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/make_dafsa.py b/src/make_dafsa.py
index 0456191..bb308f6 100755
--- a/src/make_dafsa.py
+++ b/src/make_dafsa.py
@@ -211,7 +211,7 @@ def to_dafsa(words):
     if not 0x1F < ord(word[0]) < 0x80:
       raise InputError('Domain names must be printable 7-bit ASCII')
     if len(word) == 1:
-      return chr(ord(word[0]) & 0x0F), [None]
+      return chr(int(word[0], 16) & 0x0F), [None]
     return word[0], [to_nodes(word[1:])]
   return [to_nodes(word) for word in words]
 

From 5c159a3a09b62dcc471b410d07df04002c36d14e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20R=C3=BChsen?= <tim.ruehsen@gmx.de>
Date: Fri, 1 Jan 2016 22:34:55 +0100
Subject: [PATCH 32/59] Add few more tests to test-is-public-builtin.c

---
 tests/test-is-public-builtin.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tests/test-is-public-builtin.c b/tests/test-is-public-builtin.c
index 70603ea..a38f284 100644
--- a/tests/test-is-public-builtin.c
+++ b/tests/test-is-public-builtin.c
@@ -80,6 +80,10 @@ static void test_psl(void)
 		{ ".forgot.his.name", 1 },
 		{ "whoever.his.name", 0 },
 		{ "whoever.forgot.his.name", 0 },
+		{ "whatever.platform.sh", 1 },
+		{ ".platform.sh", 1 },
+		{ "whatever.yokohama.jp", 1 },
+		{ ".yokohama.jp", 1 },
 		{ ".", 1 }, /* special case */
 		{ "", 1 },  /* special case */
 		{ NULL, 1 },  /* special case */

From 7a859d11c85418629525063d5c87139be5601a1c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20R=C3=BChsen?= <tim.ruehsen@gmx.de>
Date: Fri, 1 Jan 2016 22:35:35 +0100
Subject: [PATCH 33/59] Add PSL_TYPE* defines to docs

---
 docs/libpsl/libpsl-sections.txt | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/libpsl/libpsl-sections.txt b/docs/libpsl/libpsl-sections.txt
index 7dede1f..d72ab64 100644
--- a/docs/libpsl/libpsl-sections.txt
+++ b/docs/libpsl/libpsl-sections.txt
@@ -6,6 +6,9 @@ PSL_VERSION_MAJOR
 PSL_VERSION_MINOR
 PSL_VERSION_NUMBER
 PSL_VERSION_PATCH
+PSL_TYPE_ICANN
+PSL_TYPE_PRIVATE
+PSL_TYPE_ANY
 psl_error_t
 psl_ctx_t
 psl_load_file

From 748e3ae9cc6f6e487a81395cf8e60ff3f9d9b7c5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20R=C3=BChsen?= <tim.ruehsen@gmx.de>
Date: Fri, 1 Jan 2016 22:38:21 +0100
Subject: [PATCH 34/59] Load DAFSA precompiled files (auto-detection)

---
 src/psl.c | 50 ++++++++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 46 insertions(+), 4 deletions(-)

diff --git a/src/psl.c b/src/psl.c
index fb35c2f..ea08c76 100644
--- a/src/psl.c
+++ b/src/psl.c
@@ -171,6 +171,10 @@ typedef struct {
 struct _psl_ctx_st {
 	_psl_vector_t
 		*suffixes;
+	unsigned char
+		*dafsa;
+	size_t
+		dafsa_size;
 	int
 		mode,
 		nsuffixes,
@@ -799,7 +803,7 @@ static int _psl_is_public_suffix(const psl_ctx_t *psl, const char *domain, int t
 	for (p = domain; *p; p++) {
 		if (*p == '.')
 			suffix.nlabels++;
-		else if (*((unsigned char *)p) < 128)
+		else if (*((unsigned char *)p) >= 128)
 			need_conversion = 1; /* in case domain is non-ascii we need a toASCII conversion */
 	}
 
@@ -818,6 +822,7 @@ static int _psl_is_public_suffix(const psl_ctx_t *psl, const char *domain, int t
 			suffix.length = strlen(punycode);
 		} else {
 			/* fallback */
+
 			suffix.label = domain;
 			suffix.length = p - suffix.label;
 		}
@@ -828,8 +833,10 @@ static int _psl_is_public_suffix(const psl_ctx_t *psl, const char *domain, int t
 		suffix.length = p - suffix.label;
 	}
 
-	if (psl == &_builtin_psl) {
-		int rc = LookupStringInFixedSet(kDafsa, sizeof(kDafsa), suffix.label, suffix.length);
+	if (psl == &_builtin_psl || psl->dafsa) {
+		size_t dafsa_size = psl == &_builtin_psl ? sizeof(kDafsa) : psl->dafsa_size;
+		const unsigned char *dafsa = psl == &_builtin_psl ? kDafsa : psl->dafsa;
+		int rc = LookupStringInFixedSet(dafsa, dafsa_size, suffix.label, suffix.length);
 		if (rc != -1) {
 			/* check for correct rule type */
 			if (type == PSL_TYPE_ICANN && !(rc & _PSL_FLAG_ICANN))
@@ -849,7 +856,7 @@ static int _psl_is_public_suffix(const psl_ctx_t *psl, const char *domain, int t
 			suffix.length = strlen(suffix.label);
 			suffix.nlabels--;
 
-			rc = LookupStringInFixedSet(kDafsa, sizeof(kDafsa), suffix.label, suffix.length);
+			rc = LookupStringInFixedSet(dafsa, dafsa_size, suffix.label, suffix.length);
 			if (rc != -1) {
 				/* check for correct rule type */
 				if (type == PSL_TYPE_ICANN && !(rc & _PSL_FLAG_ICANN))
@@ -1103,6 +1110,7 @@ psl_ctx_t *psl_load_fp(FILE *fp)
 	psl_ctx_t *psl;
 	_psl_entry_t suffix, *suffixp;
 	char buf[256], *linep, *p;
+	size_t n;
 	int type = 0;
 	_psl_idna_t *idna;
 
@@ -1112,6 +1120,35 @@ psl_ctx_t *psl_load_fp(FILE *fp)
 	if (!(psl = calloc(1, sizeof(psl_ctx_t))))
 		return NULL;
 
+	/* read first line to allow ASCII / DAFSA detection */
+	if ((n = fread(buf, 1, sizeof(buf) - 1, fp)) < 1)
+		goto fail;
+
+	buf[n] = 0;
+
+	if (!strstr(buf, "This Source Code Form is subject to")) {
+		size_t size = 65536, len = n;
+
+		if (!(psl->dafsa = malloc(size)))
+			goto fail;
+
+		memcpy(psl->dafsa, buf, len);
+
+		while ((n = fread(psl->dafsa + len, 1, size - len, fp)) > 0) {
+			len += n;
+			if (len >= size) {
+				void *m = realloc(psl->dafsa, size *= 2);
+				if (!m)
+					goto fail;
+				psl->dafsa = m;
+			}
+		}
+
+		return psl;
+	}
+
+	rewind(fp);
+
 	idna = _psl_idna_open();
 
 	/*
@@ -1198,6 +1235,10 @@ psl_ctx_t *psl_load_fp(FILE *fp)
 	_psl_idna_close(idna);
 
 	return psl;
+
+fail:
+	psl_free(psl);
+	return NULL;
 }
 
 /**
@@ -1213,6 +1254,7 @@ void psl_free(psl_ctx_t *psl)
 {
 	if (psl && psl != &_builtin_psl) {
 		_vector_free(&psl->suffixes);
+		free(psl->dafsa);
 		free(psl);
 	}
 }

From 9f420e2d91591299c7c03591e8b93dd8d58628c7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20R=C3=BChsen?= <tim.ruehsen@gmx.de>
Date: Fri, 1 Jan 2016 23:26:44 +0100
Subject: [PATCH 35/59] Add tests against the built-in PSL to
 test-is-public-all.c

---
 tests/test-is-public-all.c | 148 +++++++++++++++++++++----------------
 1 file changed, 83 insertions(+), 65 deletions(-)

diff --git a/tests/test-is-public-all.c b/tests/test-is-public-all.c
index b04ccdb..bbff23d 100644
--- a/tests/test-is-public-all.c
+++ b/tests/test-is-public-all.c
@@ -54,17 +54,94 @@ static inline int _isspace_ascii(const char c)
 	return c == ' ' || c == '\t' || c == '\r' || c == '\n';
 }
 
+static void test_psl_entry(const psl_ctx_t *psl, const char *domain, int type)
+{
+	int result;
+
+	if (*domain == '!') { /* an exception to a wildcard, e.g. !www.ck (wildcard is *.ck) */
+		if ((result = psl_is_public_suffix(psl, domain + 1))) {
+			failed++;
+			printf("psl_is_public_suffix(%s)=%d (expected 0)\n", domain, result);
+		} else ok++;
+
+		if ((domain = strchr(domain, '.'))) {
+			if (!(result = psl_is_public_suffix(psl, domain + 1))) {
+				failed++;
+				printf("psl_is_public_suffix(%s)=%d (expected 1)\n", domain + 1, result);
+			} else ok++;
+		}
+	} else if (*domain == '*') { /* a wildcard, e.g. *.ck or *.platform.sh */
+		char *xdomain;
+		size_t len;
+
+		if (!(result = psl_is_public_suffix(psl, domain + 1))) {
+			failed++;
+			printf("psl_is_public_suffix(%s)=%d (expected 1)\n", domain + 1, result);
+		} else ok++;
+
+		len = strlen(domain);
+		xdomain = alloca(len + 1);
+		memcpy(xdomain, domain, len + 1);
+		*xdomain = 'x';
+		if (!(result = psl_is_public_suffix(psl, domain))) {
+			failed++;
+			printf("psl_is_public_suffix(%s)=%d (expected 1)\n", domain, result);
+		} else ok++;
+	} else {
+		if (!(result = psl_is_public_suffix(psl, domain))) {
+			failed++;
+			printf("psl_is_public_suffix(%s)=%d (expected 1)\n", domain, result);
+		} else ok++;
+
+		if (!(strchr(domain, '.'))) {
+			/* TLDs are always expected to be Publix Suffixes */
+			if (!(result = psl_is_public_suffix2(psl, domain, PSL_TYPE_PRIVATE))) {
+				failed++;
+				printf("psl_is_public_suffix2(%s, PSL_TYPE_PRIVATE)=%d (expected 1)\n", domain, result);
+			} else ok++;
+
+			if (!(result = psl_is_public_suffix2(psl, domain, PSL_TYPE_ICANN))) {
+				failed++;
+				printf("psl_is_public_suffix2(%s, PSL_TYPE_ICANN)=%d (expected 0)\n", domain, result);
+			} else ok++;
+		} else if (type == PSL_TYPE_PRIVATE) {
+			if (!(result = psl_is_public_suffix2(psl, domain, PSL_TYPE_PRIVATE))) {
+				failed++;
+				printf("psl_is_public_suffix2(%s, PSL_TYPE_PRIVATE)=%d (expected 1)\n", domain, result);
+			} else ok++;
+
+			if ((result = psl_is_public_suffix2(psl, domain, PSL_TYPE_ICANN))) {
+				failed++;
+				printf("psl_is_public_suffix2(%s, PSL_TYPE_ICANN)=%d (expected 0)\n", domain, result);
+			} else ok++;
+		} else if (type == PSL_TYPE_ICANN) {
+			if (!(result = psl_is_public_suffix2(psl, domain, PSL_TYPE_ICANN))) {
+				failed++;
+				printf("psl_is_public_suffix2(%s, PSL_TYPE_ICANN)=%d (expected 1)\n", domain, result);
+			} else ok++;
+
+			if ((result = psl_is_public_suffix2(psl, domain, PSL_TYPE_PRIVATE))) {
+				failed++;
+				printf("psl_is_public_suffix2(%s, PSL_TYPE_PRIVATE)=%d (expected 0)\n", domain, result);
+			} else ok++;
+		}
+	}
+}
+
 static void test_psl(void)
 {
 	FILE *fp;
 	psl_ctx_t *psl;
-	int result, type = 0;
+	const psl_ctx_t *psl2;
+	int type = 0;
 	char buf[256], *linep, *p;
 
 	psl = psl_load_file(PSL_FILE); /* PSL_FILE can be set by ./configure --with-psl-file=[PATH] */
-
 	printf("loaded %d suffixes and %d exceptions\n", psl_suffix_count(psl), psl_suffix_exception_count(psl));
 
+	psl2 = psl_builtin();
+	printf("builtin PSL has %d suffixes and %d exceptions\n", psl_suffix_count(psl2), psl_suffix_exception_count(psl2));
+
 	if ((fp = fopen(PSL_FILE, "r"))) {
 #ifdef HAVE_CLOCK_GETTIME
 		clock_gettime(CLOCK_REALTIME, &ts1);
@@ -93,70 +170,10 @@ static void test_psl(void)
 			for (p = linep; *linep && !_isspace_ascii(*linep);) linep++;
 			*linep = 0;
 
-			if (*p == '!') { /* an exception to a wildcard, e.g. !www.ck (wildcard is *.ck) */
-				if ((result = psl_is_public_suffix(psl, p + 1))) {
-					failed++;
-					printf("psl_is_public_suffix(%s)=%d (expected 0)\n", p, result);
-				} else ok++;
+			test_psl_entry(psl, p, type);
 
-				if ((p = strchr(p, '.'))) {
-					if (!(result = psl_is_public_suffix(psl, p + 1))) {
-						failed++;
-						printf("psl_is_public_suffix(%s)=%d (expected 1)\n", p + 1, result);
-					} else ok++;
-				}
-			}
-			else if (*p == '*') { /* a wildcard, e.g. *.ck */
-				if (!(result = psl_is_public_suffix(psl, p + 1))) {
-					failed++;
-					printf("psl_is_public_suffix(%s)=%d (expected 1)\n", p + 1, result);
-				} else ok++;
-
-				*p = 'x';
-				if (!(result = psl_is_public_suffix(psl, p))) {
-					failed++;
-					printf("psl_is_public_suffix(%s)=%d (expected 1)\n", p, result);
-				} else ok++;
-			}
-			else {
-				if (!(result = psl_is_public_suffix(psl, p))) {
-					failed++;
-					printf("psl_is_public_suffix(%s)=%d (expected 1)\n", p, result);
-				} else ok++;
-
-				if (!(strchr(p, '.'))) {
-					/* TLDs are always expected to be Publix Suffixes */
-					if (!(result = psl_is_public_suffix2(psl, p, PSL_TYPE_PRIVATE))) {
-						failed++;
-						printf("psl_is_public_suffix2(%s, PSL_TYPE_PRIVATE)=%d (expected 1)\n", p, result);
-					} else ok++;
-
-					if (!(result = psl_is_public_suffix2(psl, p, PSL_TYPE_ICANN))) {
-						failed++;
-						printf("psl_is_public_suffix2(%s, PSL_TYPE_ICANN)=%d (expected 0)\n", p, result);
-					} else ok++;
-				} else if (type == PSL_TYPE_PRIVATE) {
-					if (!(result = psl_is_public_suffix2(psl, p, PSL_TYPE_PRIVATE))) {
-						failed++;
-						printf("psl_is_public_suffix2(%s, PSL_TYPE_PRIVATE)=%d (expected 1)\n", p, result);
-					} else ok++;
-
-					if ((result = psl_is_public_suffix2(psl, p, PSL_TYPE_ICANN))) {
-						failed++;
-						printf("psl_is_public_suffix2(%s, PSL_TYPE_ICANN)=%d (expected 0)\n", p, result);
-					} else ok++;
-				} else if (type == PSL_TYPE_ICANN) {
-					if (!(result = psl_is_public_suffix2(psl, p, PSL_TYPE_ICANN))) {
-						failed++;
-						printf("psl_is_public_suffix2(%s, PSL_TYPE_ICANN)=%d (expected 1)\n", p, result);
-					} else ok++;
-
-					if ((result = psl_is_public_suffix2(psl, p, PSL_TYPE_PRIVATE))) {
-						failed++;
-						printf("psl_is_public_suffix2(%s, PSL_TYPE_PRIVATE)=%d (expected 0)\n", p, result);
-					} else ok++;
-				}
-			}
+			if (psl2)
+				test_psl_entry(psl2, p, type);
 		}
 
 #ifdef HAVE_CLOCK_GETTIME
@@ -169,6 +186,7 @@ static void test_psl(void)
 	}
 
 	psl_free(psl);
+	psl_free((psl_ctx_t *)psl2);
 }
 
 int main(int argc, const char * const *argv)

From f829668e0d4e4d916eca0edc08dc601ac1d65d19 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20R=C3=BChsen?= <tim.ruehsen@gmx.de>
Date: Sat, 2 Jan 2016 13:01:19 +0100
Subject: [PATCH 36/59] Let autogen.sh work on Solaris and without gtkdocize

---
 autogen.sh | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/autogen.sh b/autogen.sh
index e714cf8..aee0a7c 100755
--- a/autogen.sh
+++ b/autogen.sh
@@ -1,21 +1,21 @@
-#!/bin/sh -e
+#!/bin/sh
 
-AUTORECONF=$(which autoreconf 2>/dev/null || true)
+AUTORECONF=$(which autoreconf 2>/dev/null)
 if test $? -ne 0; then
   echo "No 'autoreconf' found. You must install the autoconf package."
   exit 1
 fi
 
-GIT=$(which git 2>/dev/null || true)
+GIT=$(which git 2>/dev/null)
 if test $? -ne 0; then
   echo "No 'git' found. You must install the git package."
   exit 1
 fi
 
-# create m4 before gtkdocize
-mkdir m4 2>/dev/null || true
+#  create m4 before gtkdocize
+mkdir -p m4 2>/dev/null
 
-GTKDOCIZE=$(which gtkdocize 2>/dev/null || true)
+GTKDOCIZE=$(which gtkdocize 2>/dev/null)
 if test $? -ne 0; then
   echo "No gtk-doc support found. You can't build the docs."
   # rm because gtk-doc.make might be a link to a protected file
@@ -24,7 +24,7 @@ if test $? -ne 0; then
   echo "CLEANFILES =" >>gtk-doc.make
   GTKDOCIZE=""
 else
-  $GTKDOCIZE || exit $?
+  $GTKDOCIZE
 fi
 
 $GIT submodule init

From 96e0848d811ff79c765d84ea5b82ca075ceba282 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20R=C3=BChsen?= <tim.ruehsen@gmx.de>
Date: Sat, 2 Jan 2016 13:31:53 +0100
Subject: [PATCH 37/59] Release unused memory after loading DAFSA data

---
 src/psl.c | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/psl.c b/src/psl.c
index ea08c76..10947e7 100644
--- a/src/psl.c
+++ b/src/psl.c
@@ -1127,6 +1127,7 @@ psl_ctx_t *psl_load_fp(FILE *fp)
 	buf[n] = 0;
 
 	if (!strstr(buf, "This Source Code Form is subject to")) {
+		void *m;
 		size_t size = 65536, len = n;
 
 		if (!(psl->dafsa = malloc(size)))
@@ -1137,13 +1138,16 @@ psl_ctx_t *psl_load_fp(FILE *fp)
 		while ((n = fread(psl->dafsa + len, 1, size - len, fp)) > 0) {
 			len += n;
 			if (len >= size) {
-				void *m = realloc(psl->dafsa, size *= 2);
-				if (!m)
+				if (!(m = realloc(psl->dafsa, size *= 2)))
 					goto fail;
 				psl->dafsa = m;
 			}
 		}
 
+		/* release unused memory */
+		if ((m = realloc(psl->dafsa, len)))
+			psl->dafsa = m;
+
 		return psl;
 	}
 

From 95a5152e5667f97300bb109348349dc2577c7a72 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20R=C3=BChsen?= <tim.ruehsen@gmx.de>
Date: Sat, 2 Jan 2016 13:36:49 +0100
Subject: [PATCH 38/59] Update copyright year to 2016

---
 include/libpsl.h.in            | 2 +-
 src/psl.c                      | 2 +-
 src/psl2c.c                    | 2 +-
 tests/test-is-public-all.c     | 2 +-
 tests/test-is-public-builtin.c | 2 +-
 tools/psl.c                    | 2 +-
 6 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/include/libpsl.h.in b/include/libpsl.h.in
index 59fa3f6..467021f 100644
--- a/include/libpsl.h.in
+++ b/include/libpsl.h.in
@@ -1,5 +1,5 @@
 /*
- * Copyright(c) 2014-2015 Tim Ruehsen
+ * Copyright(c) 2014-2016 Tim Ruehsen
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
diff --git a/src/psl.c b/src/psl.c
index 10947e7..f8d1462 100644
--- a/src/psl.c
+++ b/src/psl.c
@@ -1,5 +1,5 @@
 /*
- * Copyright(c) 2014-2015 Tim Ruehsen
+ * Copyright(c) 2014-2016 Tim Ruehsen
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
diff --git a/src/psl2c.c b/src/psl2c.c
index 29fdc4e..8531d40 100644
--- a/src/psl2c.c
+++ b/src/psl2c.c
@@ -1,5 +1,5 @@
 /*
- * Copyright(c) 2014-2015 Tim Ruehsen
+ * Copyright(c) 2014-2016 Tim Ruehsen
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
diff --git a/tests/test-is-public-all.c b/tests/test-is-public-all.c
index bbff23d..f4b7494 100644
--- a/tests/test-is-public-all.c
+++ b/tests/test-is-public-all.c
@@ -1,5 +1,5 @@
 /*
- * Copyright(c) 2014-2015 Tim Ruehsen
+ * Copyright(c) 2014-2016 Tim Ruehsen
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
diff --git a/tests/test-is-public-builtin.c b/tests/test-is-public-builtin.c
index a38f284..df1cb84 100644
--- a/tests/test-is-public-builtin.c
+++ b/tests/test-is-public-builtin.c
@@ -1,5 +1,5 @@
 /*
- * Copyright(c) 2014-2015 Tim Ruehsen
+ * Copyright(c) 2014-2016 Tim Ruehsen
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
diff --git a/tools/psl.c b/tools/psl.c
index 4303a5b..ec21a76 100644
--- a/tools/psl.c
+++ b/tools/psl.c
@@ -1,5 +1,5 @@
 /*
- * Copyright(c) 2014-2015 Tim Ruehsen
+ * Copyright(c) 2014-2016 Tim Ruehsen
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),

From 5bd2db75d201fc770ad110bb8b26ce47a8988490 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20R=C3=BChsen?= <tim.ruehsen@gmx.de>
Date: Sat, 2 Jan 2016 14:25:44 +0100
Subject: [PATCH 39/59] Release v0.12.0

---
 NEWS         | 15 ++++++++++++++-
 configure.ac |  4 ++--
 2 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/NEWS b/NEWS
index 97b270d..57c9b95 100644
--- a/NEWS
+++ b/NEWS
@@ -1,4 +1,17 @@
-Copyright (C) 2014-2015 Tim Rühsen
+Copyright (C) 2014-2016 Tim Rühsen
+
+02.01.2016 Release V0.12.0
+  * Load DAFSA binaries via psl_load_file() via auto-detection
+  * Add more tests
+  * Remove psl_builtin_compile_time()
+  * Compile PSL into DAFSA using make_dafsa.py
+  * Avoid libicu dependency with --enable-runtime=no
+  * Test on new Travis-CI build farm
+  * Use DAFSA format for builtin PSL data
+  * Add function psl_is_public_suffix2()
+  * Fix psl_builtin_outdated()
+  * Fix several bugs
+  * Cleanup code
 
 23.09.2015 Release V0.11.0
   * Add new function psl_check_version_number()
diff --git a/configure.ac b/configure.ac
index 9d29105..e7a1273 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1,5 +1,5 @@
 
-AC_INIT([libpsl], [0.11.0], [tim.ruehsen@gmx.de], [libpsl], [http://github.com/rockdaboot/libpsl])
+AC_INIT([libpsl], [0.12.0], [tim.ruehsen@gmx.de], [libpsl], [http://github.com/rockdaboot/libpsl])
 AC_PREREQ([2.59])
 AM_INIT_AUTOMAKE([1.10 -Wall no-define foreign])
 
@@ -85,7 +85,7 @@ PKG_PROG_PKG_CONFIG
 # 4. If any interfaces have been added, removed, or changed since the last update, increment current, and set revision to 0.
 # 5. If any interfaces have been added since the last public release, then increment age.
 # 6. If any existing interfaces have been removed or changed since the last public release, then set age to 0. 
-AC_SUBST([LIBPSL_SO_VERSION], [4:0:4])
+AC_SUBST([LIBPSL_SO_VERSION], [5:0:0])
 AC_SUBST([LIBPSL_VERSION], $VERSION)
 
 # Check for enable/disable builtin PSL data

From fd928da46e26b5f983afdeba2b80685b615dbae1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20R=C3=BChsen?= <tim.ruehsen@gmx.de>
Date: Mon, 4 Jan 2016 20:22:13 +0100
Subject: [PATCH 40/59] Fix python3 incompatibilities in make_dafsa.py

---
 src/make_dafsa.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/make_dafsa.py b/src/make_dafsa.py
index bb308f6..41df88b 100755
--- a/src/make_dafsa.py
+++ b/src/make_dafsa.py
@@ -501,7 +501,7 @@ def parse_psl(infile):
       line = line[1:]
     elif line[0] == '*':
       if line[1] != '.':
-        print 'Unsupported kind of rule (ignored): %s' % line
+        print('Unsupported kind of rule (ignored): %s', line)
         continue
       flags = PSL_FLAG_WILDCARD | PSL_FLAG_PLAIN | section
       line = line[2:]
@@ -536,11 +536,11 @@ def parse_psl(infile):
 
 def usage():
   """Prints the usage"""
-  print 'usage: %s [options] infile outfile' % sys.argv[0]
-  print '  --input-format=psl2c    infile has been generated by libpsl/psl2c utility (default)'
-  print '  --input-format=psl      infile is a Public Suffix List file'
-  print '  --output-format=cxx     Write DAFSA as C/C++ code'
-  print '  --output-format=binary  Write DAFSA binary data'
+  print('usage: %s [options] infile outfile', sys.argv[0])
+  print('  --input-format=psl2c    infile has been generated by libpsl/psl2c utility (default)')
+  print('  --input-format=psl      infile is a Public Suffix List file')
+  print('  --output-format=cxx     Write DAFSA as C/C++ code')
+  print('  --output-format=binary  Write DAFSA binary data')
   exit(1)
 
 
@@ -560,7 +560,7 @@ def main():
       elif value == 'psl2c':
         parser = parse_psl2c
       else:
-        print "Unknown input format '%s'" % value
+        print("Unknown input format '%s'", value)
         return 1
     elif arg.startswith('--output-format='):
       value = arg[16:].lower()
@@ -569,7 +569,7 @@ def main():
       elif value == 'cxx':
         converter = words_to_cxx
       else:
-        print "Unknown output format '%s'" % value
+        print("Unknown output format '%s'", value)
         return 1
     else:
       usage()

From f86094972ae8efd4d1cb5aac3cf393fcb544187f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20R=C3=BChsen?= <tim.ruehsen@gmx.de>
Date: Mon, 4 Jan 2016 20:41:24 +0100
Subject: [PATCH 41/59] Check for python 2.7+ in configure.ac

---
 README.md    | 2 ++
 configure.ac | 3 +++
 2 files changed, 5 insertions(+)

diff --git a/README.md b/README.md
index 75e1038..8ed0b52 100644
--- a/README.md
+++ b/README.md
@@ -82,6 +82,8 @@ src/LICENSE.chromium.
 Building from git
 -----------------
 
+You should have python2.7+ installed.
+
 Download project and prepare sources with
 
 		git clone http://github.com/rockdaboot/libpsl
diff --git a/configure.ac b/configure.ac
index e7a1273..a14f4ec 100644
--- a/configure.ac
+++ b/configure.ac
@@ -72,6 +72,9 @@ AS_IF([ test "$enable_man" != no ], [
   AC_MSG_RESULT([no])
 ])
 
+# src/make_dafsa.py needs python 2.7+
+AM_PATH_PYTHON([2.7])
+
 PKG_PROG_PKG_CONFIG
 
 # Define these substitions here to keep all version information in one place.

From 1bd9347af93edf1e50141518ca2b2edced9e2676 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20R=C3=BChsen?= <tim.ruehsen@gmx.de>
Date: Mon, 4 Jan 2016 22:15:43 +0100
Subject: [PATCH 42/59] Fix for commit fd928da46e26b5f983afdeba2b80685b615dbae1

---
 src/make_dafsa.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/make_dafsa.py b/src/make_dafsa.py
index 41df88b..a656ad2 100755
--- a/src/make_dafsa.py
+++ b/src/make_dafsa.py
@@ -501,7 +501,7 @@ def parse_psl(infile):
       line = line[1:]
     elif line[0] == '*':
       if line[1] != '.':
-        print('Unsupported kind of rule (ignored): %s', line)
+        print('Unsupported kind of rule (ignored): %s' % line)
         continue
       flags = PSL_FLAG_WILDCARD | PSL_FLAG_PLAIN | section
       line = line[2:]
@@ -536,7 +536,7 @@ def parse_psl(infile):
 
 def usage():
   """Prints the usage"""
-  print('usage: %s [options] infile outfile', sys.argv[0])
+  print('usage: %s [options] infile outfile' % sys.argv[0])
   print('  --input-format=psl2c    infile has been generated by libpsl/psl2c utility (default)')
   print('  --input-format=psl      infile is a Public Suffix List file')
   print('  --output-format=cxx     Write DAFSA as C/C++ code')
@@ -560,7 +560,7 @@ def main():
       elif value == 'psl2c':
         parser = parse_psl2c
       else:
-        print("Unknown input format '%s'", value)
+        print("Unknown input format '%s'" % value)
         return 1
     elif arg.startswith('--output-format='):
       value = arg[16:].lower()
@@ -569,7 +569,7 @@ def main():
       elif value == 'cxx':
         converter = words_to_cxx
       else:
-        print("Unknown output format '%s'", value)
+        print("Unknown output format '%s'" % value)
         return 1
     else:
       usage()

From 8c39291f55311c7ebbb4ada25c07cd9992590e96 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20R=C3=BChsen?= <tim.ruehsen@gmx.de>
Date: Tue, 5 Jan 2016 10:57:07 +0100
Subject: [PATCH 43/59] Slightly shorter DAFSA array when sorting input

---
 src/make_dafsa.py | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/src/make_dafsa.py b/src/make_dafsa.py
index a656ad2..837468a 100755
--- a/src/make_dafsa.py
+++ b/src/make_dafsa.py
@@ -458,14 +458,13 @@ def parse_psl2c(infile):
     # Technically the DAFSA format could support return values in range [0-31],
     # but the values below are the only with a defined meaning.
     if line[-1] not in '0123456789ABCDEF':
-      raise InputError('Expected value to be one of {0,1,2,4,5}, found "%s"' %
-                       line[-1])
+      raise InputError('Expected value to be one of {0,1,2,4,5}, found "%s"' % line[-1])
 
 #  with open("gperf.out", 'w') as outfile:
-#    for line in lines:
+#    for line in sorted(lines):
 #      outfile.write(line[:-3] + line[-1] + "\n")
 
-  return [line[:-3] + line[-1] for line in lines]
+  return [line[:-3] + line[-1] for line in sorted(lines)]
 
 
 def parse_psl(infile):
@@ -528,10 +527,10 @@ def parse_psl(infile):
     psl[line] = flags
 
 #  with open("psl.out", 'w') as outfile:
-#    for (domain, flags) in psl.iteritems():
+#    for (domain, flags) in sorted(psl.iteritems()):
 #      outfile.write(domain + "%X" % (flags & 0x0F) + "\n")
 
-  return [domain + "%X" % (flags & 0x0F) for (domain, flags) in psl.iteritems()]
+  return [domain + "%X" % (flags & 0x0F) for (domain, flags) in sorted(psl.iteritems())]
 
 
 def usage():

From 3ba8903915a9d14ddf1db0e08456e1445265f34e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20R=C3=BChsen?= <tim.ruehsen@gmx.de>
Date: Tue, 2 Feb 2016 16:43:03 +0100
Subject: [PATCH 44/59] Add PSL linter written in Python

---
 src/pslint.py | 185 ++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 185 insertions(+)
 create mode 100755 src/pslint.py

diff --git a/src/pslint.py b/src/pslint.py
new file mode 100755
index 0000000..1d677d4
--- /dev/null
+++ b/src/pslint.py
@@ -0,0 +1,185 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-#
+#
+# PSL linter written in python
+#
+# Copyright 2016 Tim Rühsen (tim dot ruehsen at gmx dot de). All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+import sys
+
+nline = 0
+line = ""
+warnings = 0
+errors = 0
+
+def warning(msg):
+  global warnings, line, nline
+  print('%d: warning: %s: \'%s\'' % (nline, msg, line))
+  warnings += 1
+
+def error(msg):
+  global errors, line, nline
+  print('%d: error: %s: \'%s\'' % (nline, msg, line))
+  errors += 1
+
+def lint_psl(infile):
+  """Parses PSL file and extract strings and return code"""
+  global line, nline
+
+  PSL_FLAG_EXCEPTION = (1<<0)
+  PSL_FLAG_WILDCARD = (1<<1)
+  PSL_FLAG_ICANN = (1<<2) # entry of ICANN section
+  PSL_FLAG_PRIVATE = (1<<3) # entry of PRIVATE section
+  PSL_FLAG_PLAIN = (1<<4) #just used for PSL syntax checking
+
+  psl = {}
+  section = 0
+
+  lines = [line.strip('\r\n') for line in infile]
+
+  for line in lines:
+    nline += 1
+
+	 # check for leadind/trailing whitespace
+    stripped = line.strip()
+    if stripped != line:
+      warning('Leading/Trailing whitespace')
+    line = stripped
+
+    # empty line
+    if not line:
+      continue
+
+    # check for section begin/end
+    if line[0:2] == "//":
+      if section == 0:
+        if line == "// ===BEGIN ICANN DOMAINS===":
+          section = PSL_FLAG_ICANN
+        elif line == "// ===BEGIN PRIVATE DOMAINS===":
+          section = PSL_FLAG_PRIVATE
+        elif line[3:8] == "===BEGIN":
+          error('Unexpected begin of unknown section')
+        elif line[3:6] == "===END":
+          error('End of section without previous begin')
+      elif section == PSL_FLAG_ICANN:
+        if line == "// ===END ICANN DOMAINS===":
+          section = 0
+        elif line[3:8] == "===BEGIN":
+          error('Unexpected begin of section: ')
+        elif line[3:6] == "===END":
+          error('Unexpected end of section')
+      elif section == PSL_FLAG_PRIVATE:
+        if line == "// ===END ICANN DOMAINS===":
+          section = 0
+        elif line[3:8] == "===BEGIN":
+          error('Unexpected begin of section')
+        elif line[3:6] == "===END":
+          error('Unexpected end of section')
+
+      continue # processing of comments ends here
+
+    # No rule must be outside of a section
+    if section == 0:
+      error('Rule outside of section')
+
+    # decode UTF-8 input into unicode, needed only for python 2.x
+    if sys.version_info[0] < 3:
+      line = line.decode('utf-8')
+
+	 # each rule must be lowercase (or more exactly: not uppercase and not titlecase)
+    if line != line.lower():
+      error('Rule must be lowercase')
+
+    # strip leading wildcards
+    flags = 0
+    # while line[0:2] == '*.':
+    if line[0:2] == '*.':
+      flags = PSL_FLAG_WILDCARD | PSL_FLAG_PLAIN | section
+      line = line[2:]
+
+    if line[0] == '!':
+      flags = PSL_FLAG_EXCEPTION | section
+      line = line[1:]
+
+    # wildcard and exception must not combine
+    if flags & PSL_FLAG_WILDCARD and flags & PSL_FLAG_EXCEPTION:
+      error('Combination of wildcard and exception')
+
+    labels = line.split('.')
+
+    for label in labels:
+      if not label:
+         error('Leading/trailing or multiple dot')
+         continue
+
+      if label[0:4] == 'xn--':
+         error('Punycode found')
+         continue
+
+      if '--' in label:
+         error('Double minus found')
+         continue
+
+      # allowed are a-z,0-9,- and unicode >= 128 (maybe that can be finetuned a bit !?)
+      for c in label:
+        if not c.isalnum() and c != '-' and ord(c) < 128:
+          error('Illegal character')
+          break
+
+    if line in psl:
+      """Found existing entry:
+         Combination of exception and plain rule is ambiguous
+           !foo.bar
+            foo.bar
+
+         Allowed:
+           !foo.bar + *.foo.bar
+            foo.bar + *.foo.bar
+      """
+      error('Found doublette/ambiguity (previous line was %d)' % psl[line])
+      continue
+
+    psl[line] = nline
+
+
+def usage():
+  """Prints the usage"""
+  print('usage: %s PSLfile' % sys.argv[0])
+  print('or     %s -        # To read PSL from STDIN' % sys.argv[0])
+  exit(1)
+
+
+def main():
+  """Check syntax of a PSL file"""
+  if len(sys.argv) < 2:
+    usage()
+
+  if sys.argv[-1] == '-':
+    lint_psl(sys.stdin)
+  else:
+    with open(sys.argv[-1], 'r') as infile:
+      lint_psl(infile)
+
+  return errors != 0
+
+
+if __name__ == '__main__':
+  sys.exit(main())

From 98aed19c3a417d85854569686ad29f3a5a53f311 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20R=C3=BChsen?= <tim.ruehsen@gmx.de>
Date: Tue, 2 Feb 2016 19:59:45 +0100
Subject: [PATCH 45/59] Convert copyright line to UTF-8 in pslint.py

---
 list          | 2 +-
 src/pslint.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/list b/list
index 1f3ad51..15e658e 160000
--- a/list
+++ b/list
@@ -1 +1 @@
-Subproject commit 1f3ad51171235aafe423435606e869f0161582e4
+Subproject commit 15e658eeed39dc3e8cf23ca58f4324385bdf11ed
diff --git a/src/pslint.py b/src/pslint.py
index 1d677d4..37bf4df 100755
--- a/src/pslint.py
+++ b/src/pslint.py
@@ -3,7 +3,7 @@
 #
 # PSL linter written in python
 #
-# Copyright 2016 Tim Rühsen (tim dot ruehsen at gmx dot de). All rights reserved.
+# Copyright 2016 Tim Rühsen (tim dot ruehsen at gmx dot de). All rights reserved.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a
 # copy of this software and associated documentation files (the "Software"),

From bd70c79c1829a8346ca052d1d53e89a9c6cd06a1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20R=C3=BChsen?= <tim.ruehsen@gmx.de>
Date: Tue, 2 Feb 2016 22:20:58 +0100
Subject: [PATCH 46/59] Indent src/pslint.py with tabs

---
 src/pslint.py | 228 +++++++++++++++++++++++++-------------------------
 1 file changed, 114 insertions(+), 114 deletions(-)

diff --git a/src/pslint.py b/src/pslint.py
index 37bf4df..7582ae2 100755
--- a/src/pslint.py
+++ b/src/pslint.py
@@ -31,155 +31,155 @@ warnings = 0
 errors = 0
 
 def warning(msg):
-  global warnings, line, nline
-  print('%d: warning: %s: \'%s\'' % (nline, msg, line))
-  warnings += 1
+	global warnings, line, nline
+	print('%d: warning: %s: \'%s\'' % (nline, msg, line))
+	warnings += 1
 
 def error(msg):
-  global errors, line, nline
-  print('%d: error: %s: \'%s\'' % (nline, msg, line))
-  errors += 1
+	global errors, line, nline
+	print('%d: error: %s: \'%s\'' % (nline, msg, line))
+	errors += 1
 
 def lint_psl(infile):
-  """Parses PSL file and extract strings and return code"""
-  global line, nline
+	"""Parses PSL file and extract strings and return code"""
+	global line, nline
 
-  PSL_FLAG_EXCEPTION = (1<<0)
-  PSL_FLAG_WILDCARD = (1<<1)
-  PSL_FLAG_ICANN = (1<<2) # entry of ICANN section
-  PSL_FLAG_PRIVATE = (1<<3) # entry of PRIVATE section
-  PSL_FLAG_PLAIN = (1<<4) #just used for PSL syntax checking
+	PSL_FLAG_EXCEPTION = (1<<0)
+	PSL_FLAG_WILDCARD = (1<<1)
+	PSL_FLAG_ICANN = (1<<2) # entry of ICANN section
+	PSL_FLAG_PRIVATE = (1<<3) # entry of PRIVATE section
+	PSL_FLAG_PLAIN = (1<<4) #just used for PSL syntax checking
 
-  psl = {}
-  section = 0
+	psl = {}
+	section = 0
 
-  lines = [line.strip('\r\n') for line in infile]
+	lines = [line.strip('\r\n') for line in infile]
 
-  for line in lines:
-    nline += 1
+	for line in lines:
+		nline += 1
 
-	 # check for leadind/trailing whitespace
-    stripped = line.strip()
-    if stripped != line:
-      warning('Leading/Trailing whitespace')
-    line = stripped
+		# check for leadind/trailing whitespace
+		stripped = line.strip()
+		if stripped != line:
+			warning('Leading/Trailing whitespace')
+		line = stripped
 
-    # empty line
-    if not line:
-      continue
+		# empty line
+		if not line:
+			continue
 
-    # check for section begin/end
-    if line[0:2] == "//":
-      if section == 0:
-        if line == "// ===BEGIN ICANN DOMAINS===":
-          section = PSL_FLAG_ICANN
-        elif line == "// ===BEGIN PRIVATE DOMAINS===":
-          section = PSL_FLAG_PRIVATE
-        elif line[3:8] == "===BEGIN":
-          error('Unexpected begin of unknown section')
-        elif line[3:6] == "===END":
-          error('End of section without previous begin')
-      elif section == PSL_FLAG_ICANN:
-        if line == "// ===END ICANN DOMAINS===":
-          section = 0
-        elif line[3:8] == "===BEGIN":
-          error('Unexpected begin of section: ')
-        elif line[3:6] == "===END":
-          error('Unexpected end of section')
-      elif section == PSL_FLAG_PRIVATE:
-        if line == "// ===END ICANN DOMAINS===":
-          section = 0
-        elif line[3:8] == "===BEGIN":
-          error('Unexpected begin of section')
-        elif line[3:6] == "===END":
-          error('Unexpected end of section')
+		# check for section begin/end
+		if line[0:2] == "//":
+			if section == 0:
+				if line == "// ===BEGIN ICANN DOMAINS===":
+					section = PSL_FLAG_ICANN
+				elif line == "// ===BEGIN PRIVATE DOMAINS===":
+					section = PSL_FLAG_PRIVATE
+				elif line[3:8] == "===BEGIN":
+					error('Unexpected begin of unknown section')
+				elif line[3:6] == "===END":
+					error('End of section without previous begin')
+			elif section == PSL_FLAG_ICANN:
+				if line == "// ===END ICANN DOMAINS===":
+					section = 0
+				elif line[3:8] == "===BEGIN":
+					error('Unexpected begin of section: ')
+				elif line[3:6] == "===END":
+					error('Unexpected end of section')
+			elif section == PSL_FLAG_PRIVATE:
+				if line == "// ===END ICANN DOMAINS===":
+					section = 0
+				elif line[3:8] == "===BEGIN":
+					error('Unexpected begin of section')
+				elif line[3:6] == "===END":
+					error('Unexpected end of section')
 
-      continue # processing of comments ends here
+			continue # processing of comments ends here
 
-    # No rule must be outside of a section
-    if section == 0:
-      error('Rule outside of section')
+		# No rule must be outside of a section
+		if section == 0:
+			error('Rule outside of section')
 
-    # decode UTF-8 input into unicode, needed only for python 2.x
-    if sys.version_info[0] < 3:
-      line = line.decode('utf-8')
+		# decode UTF-8 input into unicode, needed only for python 2.x
+		if sys.version_info[0] < 3:
+			line = line.decode('utf-8')
 
 	 # each rule must be lowercase (or more exactly: not uppercase and not titlecase)
-    if line != line.lower():
-      error('Rule must be lowercase')
+		if line != line.lower():
+			error('Rule must be lowercase')
 
-    # strip leading wildcards
-    flags = 0
-    # while line[0:2] == '*.':
-    if line[0:2] == '*.':
-      flags = PSL_FLAG_WILDCARD | PSL_FLAG_PLAIN | section
-      line = line[2:]
+		# strip leading wildcards
+		flags = 0
+		# while line[0:2] == '*.':
+		if line[0:2] == '*.':
+			flags = PSL_FLAG_WILDCARD | PSL_FLAG_PLAIN | section
+			line = line[2:]
 
-    if line[0] == '!':
-      flags = PSL_FLAG_EXCEPTION | section
-      line = line[1:]
+		if line[0] == '!':
+			flags = PSL_FLAG_EXCEPTION | section
+			line = line[1:]
 
-    # wildcard and exception must not combine
-    if flags & PSL_FLAG_WILDCARD and flags & PSL_FLAG_EXCEPTION:
-      error('Combination of wildcard and exception')
+		# wildcard and exception must not combine
+		if flags & PSL_FLAG_WILDCARD and flags & PSL_FLAG_EXCEPTION:
+			error('Combination of wildcard and exception')
 
-    labels = line.split('.')
+		labels = line.split('.')
 
-    for label in labels:
-      if not label:
-         error('Leading/trailing or multiple dot')
-         continue
+		for label in labels:
+			if not label:
+				 error('Leading/trailing or multiple dot')
+				 continue
 
-      if label[0:4] == 'xn--':
-         error('Punycode found')
-         continue
+			if label[0:4] == 'xn--':
+				 error('Punycode found')
+				 continue
 
-      if '--' in label:
-         error('Double minus found')
-         continue
+			if '--' in label:
+				 error('Double minus found')
+				 continue
 
-      # allowed are a-z,0-9,- and unicode >= 128 (maybe that can be finetuned a bit !?)
-      for c in label:
-        if not c.isalnum() and c != '-' and ord(c) < 128:
-          error('Illegal character')
-          break
+			# allowed are a-z,0-9,- and unicode >= 128 (maybe that can be finetuned a bit !?)
+			for c in label:
+				if not c.isalnum() and c != '-' and ord(c) < 128:
+					error('Illegal character')
+					break
 
-    if line in psl:
-      """Found existing entry:
-         Combination of exception and plain rule is ambiguous
-           !foo.bar
-            foo.bar
+		if line in psl:
+			"""Found existing entry:
+				 Combination of exception and plain rule is ambiguous
+					 !foo.bar
+						foo.bar
 
-         Allowed:
-           !foo.bar + *.foo.bar
-            foo.bar + *.foo.bar
-      """
-      error('Found doublette/ambiguity (previous line was %d)' % psl[line])
-      continue
+				 Allowed:
+					 !foo.bar + *.foo.bar
+						foo.bar + *.foo.bar
+			"""
+			error('Found doublette/ambiguity (previous line was %d)' % psl[line])
+			continue
 
-    psl[line] = nline
+		psl[line] = nline
 
 
 def usage():
-  """Prints the usage"""
-  print('usage: %s PSLfile' % sys.argv[0])
-  print('or     %s -        # To read PSL from STDIN' % sys.argv[0])
-  exit(1)
+	"""Prints the usage"""
+	print('usage: %s PSLfile' % sys.argv[0])
+	print('or     %s -        # To read PSL from STDIN' % sys.argv[0])
+	exit(1)
 
 
 def main():
-  """Check syntax of a PSL file"""
-  if len(sys.argv) < 2:
-    usage()
+	"""Check syntax of a PSL file"""
+	if len(sys.argv) < 2:
+		usage()
 
-  if sys.argv[-1] == '-':
-    lint_psl(sys.stdin)
-  else:
-    with open(sys.argv[-1], 'r') as infile:
-      lint_psl(infile)
+	if sys.argv[-1] == '-':
+		lint_psl(sys.stdin)
+	else:
+		with open(sys.argv[-1], 'r') as infile:
+			lint_psl(infile)
 
-  return errors != 0
+	return errors != 0
 
 
 if __name__ == '__main__':
-  sys.exit(main())
+	sys.exit(main())

From a46af675b456aaf12e5d1eb7f1ea6ef8ca68b92b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20R=C3=BChsen?= <tim.ruehsen@gmx.de>
Date: Tue, 2 Feb 2016 22:41:18 +0100
Subject: [PATCH 47/59] Fix indentation multi-line comment in src/pslint.py

---
 src/pslint.py | 30 +++++++++++++++++-------------
 1 file changed, 17 insertions(+), 13 deletions(-)

diff --git a/src/pslint.py b/src/pslint.py
index 7582ae2..22f8eb1 100755
--- a/src/pslint.py
+++ b/src/pslint.py
@@ -50,7 +50,8 @@ def lint_psl(infile):
 	PSL_FLAG_PRIVATE = (1<<3) # entry of PRIVATE section
 	PSL_FLAG_PLAIN = (1<<4) #just used for PSL syntax checking
 
-	psl = {}
+	line2number = {}
+	line2flag = {}
 	section = 0
 
 	lines = [line.strip('\r\n') for line in infile]
@@ -109,15 +110,17 @@ def lint_psl(infile):
 			error('Rule must be lowercase')
 
 		# strip leading wildcards
-		flags = 0
+		flags = section
 		# while line[0:2] == '*.':
 		if line[0:2] == '*.':
-			flags = PSL_FLAG_WILDCARD | PSL_FLAG_PLAIN | section
+			flags |= PSL_FLAG_WILDCARD
 			line = line[2:]
 
 		if line[0] == '!':
-			flags = PSL_FLAG_EXCEPTION | section
+			flags |= PSL_FLAG_EXCEPTION
 			line = line[1:]
+		else:
+			flags |= PSL_FLAG_PLAIN
 
 		# wildcard and exception must not combine
 		if flags & PSL_FLAG_WILDCARD and flags & PSL_FLAG_EXCEPTION:
@@ -144,20 +147,21 @@ def lint_psl(infile):
 					error('Illegal character')
 					break
 
-		if line in psl:
+		if line in line2number:
 			"""Found existing entry:
-				 Combination of exception and plain rule is ambiguous
-					 !foo.bar
-						foo.bar
+			   Combination of exception and plain rule is ambiguous
+			     !foo.bar
+			      foo.bar
 
-				 Allowed:
-					 !foo.bar + *.foo.bar
-						foo.bar + *.foo.bar
+			   Allowed:
+			     !foo.bar + *.foo.bar
+			      foo.bar + *.foo.bar
 			"""
-			error('Found doublette/ambiguity (previous line was %d)' % psl[line])
+			error('Found doublette/ambiguity (previous line was %d)' % line2number[line])
 			continue
 
-		psl[line] = nline
+		line2number[line] = nline
+		line2flag[line] = flags
 
 
 def usage():

From aa028e606b55249e18fb9f7cca7a66694b9d099e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20R=C3=BChsen?= <tim.ruehsen@gmx.de>
Date: Tue, 2 Feb 2016 22:49:02 +0100
Subject: [PATCH 48/59] Adjust text in doublette comment in src/pslint.py

---
 src/pslint.py | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/src/pslint.py b/src/pslint.py
index 22f8eb1..c8f8c3a 100755
--- a/src/pslint.py
+++ b/src/pslint.py
@@ -147,15 +147,14 @@ def lint_psl(infile):
 					error('Illegal character')
 					break
 
-		if line in line2number:
+		if line in line2flag:
 			"""Found existing entry:
-			   Combination of exception and plain rule is ambiguous
-			     !foo.bar
-			      foo.bar
-
+			   Combination of exception and plain rule is contradictionary
+			     !foo.bar + foo.bar
+			   Doublette, since *.foo.bar implies foo.bar:
+			      foo.bar + *.foo.bar
 			   Allowed:
 			     !foo.bar + *.foo.bar
-			      foo.bar + *.foo.bar
 			"""
 			error('Found doublette/ambiguity (previous line was %d)' % line2number[line])
 			continue

From 0e2da86eeb1cc5a01c0d6ce51bf0ef248466d5f1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20R=C3=BChsen?= <tim.ruehsen@gmx.de>
Date: Tue, 2 Feb 2016 22:52:29 +0100
Subject: [PATCH 49/59] Update PSL submodule

---
 list | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/list b/list
index 15e658e..24caf4f 160000
--- a/list
+++ b/list
@@ -1 +1 @@
-Subproject commit 15e658eeed39dc3e8cf23ca58f4324385bdf11ed
+Subproject commit 24caf4f72bf42a87559685e7211796c94855a90c

From 568394438d737f155d82bd0515b68a3c48f505a7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20R=C3=BChsen?= <tim.ruehsen@gmx.de>
Date: Fri, 5 Feb 2016 12:16:50 +0100
Subject: [PATCH 50/59] Add disabled code for 'Group Order' checking

The check has been disabled since it turned out that those
'groupings' of PSL entries are not really ordered in the way
(# of labels, TLD, sublabel#1, sublabel#2, ...)

This commit also fixes section detection / verification
---
 src/pslint.py | 79 ++++++++++++++++++++++++++++++++++++++++-----------
 1 file changed, 62 insertions(+), 17 deletions(-)

diff --git a/src/pslint.py b/src/pslint.py
index c8f8c3a..7a91bca 100755
--- a/src/pslint.py
+++ b/src/pslint.py
@@ -29,16 +29,57 @@ nline = 0
 line = ""
 warnings = 0
 errors = 0
+skip_order_check = False
 
 def warning(msg):
 	global warnings, line, nline
-	print('%d: warning: %s: \'%s\'' % (nline, msg, line))
+	print('%d: warning: %s%s' % (nline, msg, ": \'" + line + "\'" if line else ""))
 	warnings += 1
 
 def error(msg):
 	global errors, line, nline
-	print('%d: error: %s: \'%s\'' % (nline, msg, line))
+	print('%d: error: %s%s' % (nline, msg, ": \'" + line + "\'" if line else ""))
 	errors += 1
+#	skip_order_check = True
+
+def print_list(list):
+	print("  [" + ", ".join( str(x) for x in list) + "]")
+
+def psl_key(s):
+	if s[0] == '*':
+		return 0
+	if s[0] == '!':
+		return 1
+	return 0
+
+def psl_sort(group):
+	# we have to extend the inner lists for sorting/comparing
+#	needed = max(len(lables) for lables in group)
+#	return sorted(group, key = lambda labels: (len(labels), labels.extend([None] * (needed - len(labels)))))
+	return sorted(group, key = lambda labels: (len(labels), labels))
+
+def check_order(group):
+	global skip_order_check
+
+	try:
+		if skip_order_check or len(group) < 2:
+			skip_order_check = False
+			return
+
+		# check if the TLD is the identical within the group
+		if any(group[0][0] != labels[0] for labels in group):
+			error('Domain group TLD is not consistent')
+			return
+
+		sorted_group = psl_sort(group)
+		if group != sorted_group:
+			warning('Incorrectly sorted group of domains')
+			print_list(group)
+			print_list(sorted_group)
+
+	finally:
+		del group[:]
+
 
 def lint_psl(infile):
 	"""Parses PSL file and extract strings and return code"""
@@ -53,6 +94,7 @@ def lint_psl(infile):
 	line2number = {}
 	line2flag = {}
 	section = 0
+	group = []
 
 	lines = [line.strip('\r\n') for line in infile]
 
@@ -65,34 +107,37 @@ def lint_psl(infile):
 			warning('Leading/Trailing whitespace')
 		line = stripped
 
-		# empty line
+		# empty line (end of sorted domain group)
 		if not line:
+			# check_order(group)
 			continue
 
 		# check for section begin/end
 		if line[0:2] == "//":
+			# check_order(group)
+
 			if section == 0:
 				if line == "// ===BEGIN ICANN DOMAINS===":
 					section = PSL_FLAG_ICANN
 				elif line == "// ===BEGIN PRIVATE DOMAINS===":
 					section = PSL_FLAG_PRIVATE
-				elif line[3:8] == "===BEGIN":
+				elif line[3:11] == "===BEGIN":
 					error('Unexpected begin of unknown section')
-				elif line[3:6] == "===END":
+				elif line[3:9] == "===END":
 					error('End of section without previous begin')
 			elif section == PSL_FLAG_ICANN:
 				if line == "// ===END ICANN DOMAINS===":
 					section = 0
-				elif line[3:8] == "===BEGIN":
+				elif line[3:11] == "===BEGIN":
 					error('Unexpected begin of section: ')
-				elif line[3:6] == "===END":
+				elif line[3:9] == "===END":
 					error('Unexpected end of section')
 			elif section == PSL_FLAG_PRIVATE:
 				if line == "// ===END ICANN DOMAINS===":
 					section = 0
-				elif line[3:8] == "===BEGIN":
+				elif line[3:11] == "===BEGIN":
 					error('Unexpected begin of section')
-				elif line[3:6] == "===END":
+				elif line[3:9] == "===END":
 					error('Unexpected end of section')
 
 			continue # processing of comments ends here
@@ -105,7 +150,7 @@ def lint_psl(infile):
 		if sys.version_info[0] < 3:
 			line = line.decode('utf-8')
 
-	 # each rule must be lowercase (or more exactly: not uppercase and not titlecase)
+		# each rule must be lowercase (or more exactly: not uppercase and not titlecase)
 		if line != line.lower():
 			error('Rule must be lowercase')
 
@@ -128,6 +173,9 @@ def lint_psl(infile):
 
 		labels = line.split('.')
 
+		# collect reversed list of labels
+		group.append(list(reversed(line.encode('utf-8').split('.'))))
+
 		for label in labels:
 			if not label:
 				 error('Leading/trailing or multiple dot')
@@ -148,14 +196,14 @@ def lint_psl(infile):
 					break
 
 		if line in line2flag:
-			"""Found existing entry:
+			'''Found existing entry:
 			   Combination of exception and plain rule is contradictionary
 			     !foo.bar + foo.bar
 			   Doublette, since *.foo.bar implies foo.bar:
 			      foo.bar + *.foo.bar
 			   Allowed:
 			     !foo.bar + *.foo.bar
-			"""
+			'''
 			error('Found doublette/ambiguity (previous line was %d)' % line2number[line])
 			continue
 
@@ -175,11 +223,8 @@ def main():
 	if len(sys.argv) < 2:
 		usage()
 
-	if sys.argv[-1] == '-':
-		lint_psl(sys.stdin)
-	else:
-		with open(sys.argv[-1], 'r') as infile:
-			lint_psl(infile)
+	with sys.stdin if sys.argv[-1] == '-' else open(sys.argv[-1], 'r') as infile:
+		lint_psl(infile)
 
 	return errors != 0
 

From 36609787d5c14667d93cd55d716fc06263260dd9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20R=C3=BChsen?= <tim.ruehsen@gmx.de>
Date: Mon, 8 Feb 2016 09:40:43 +0100
Subject: [PATCH 51/59] Fix python3 UTF-8 runtime error and section detection

---
 list          | 2 +-
 src/pslint.py | 7 +++++--
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/list b/list
index 24caf4f..1f3ad51 160000
--- a/list
+++ b/list
@@ -1 +1 @@
-Subproject commit 24caf4f72bf42a87559685e7211796c94855a90c
+Subproject commit 1f3ad51171235aafe423435606e869f0161582e4
diff --git a/src/pslint.py b/src/pslint.py
index 7a91bca..b9104ab 100755
--- a/src/pslint.py
+++ b/src/pslint.py
@@ -133,7 +133,7 @@ def lint_psl(infile):
 				elif line[3:9] == "===END":
 					error('Unexpected end of section')
 			elif section == PSL_FLAG_PRIVATE:
-				if line == "// ===END ICANN DOMAINS===":
+				if line == "// ===END PRIVATE DOMAINS===":
 					section = 0
 				elif line[3:11] == "===BEGIN":
 					error('Unexpected begin of section')
@@ -174,7 +174,10 @@ def lint_psl(infile):
 		labels = line.split('.')
 
 		# collect reversed list of labels
-		group.append(list(reversed(line.encode('utf-8').split('.'))))
+		if sys.version_info[0] < 3:
+			group.append(list(reversed(line.encode('utf-8').split('.'))))
+		else:
+			group.append(list(reversed(line.split('.'))))
 
 		for label in labels:
 			if not label:

From d19c46c00304603f3afb6487ea0d307df9ff4135 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20R=C3=BChsen?= <tim.ruehsen@gmx.de>
Date: Mon, 8 Feb 2016 14:11:52 +0100
Subject: [PATCH 52/59] Make a few enhancements to pslint

---
 src/pslint.py | 42 +++++++++++++++++++++---------------------
 1 file changed, 21 insertions(+), 21 deletions(-)

diff --git a/src/pslint.py b/src/pslint.py
index b9104ab..6148914 100755
--- a/src/pslint.py
+++ b/src/pslint.py
@@ -42,23 +42,19 @@ def error(msg):
 	errors += 1
 #	skip_order_check = True
 
-def print_list(list):
-	print("  [" + ", ".join( str(x) for x in list) + "]")
+def print_psl(list):
+	for domain in list:
+		print(".".join(str(label) for label in reversed(domain)))
 
 def psl_key(s):
 	if s[0] == '*':
 		return 0
 	if s[0] == '!':
 		return 1
-	return 0
-
-def psl_sort(group):
-	# we have to extend the inner lists for sorting/comparing
-#	needed = max(len(lables) for lables in group)
-#	return sorted(group, key = lambda labels: (len(labels), labels.extend([None] * (needed - len(labels)))))
-	return sorted(group, key = lambda labels: (len(labels), labels))
+	return 2
 
 def check_order(group):
+	"""Check the correct order of a domain group"""
 	global skip_order_check
 
 	try:
@@ -68,21 +64,23 @@ def check_order(group):
 
 		# check if the TLD is the identical within the group
 		if any(group[0][0] != labels[0] for labels in group):
-			error('Domain group TLD is not consistent')
-			return
+			warning('Domain group TLD is not consistent')
+
+		# sort by # of labels, label-by-label (labels are in reversed order)
+		sorted_group = sorted(group, key = lambda labels: (len(labels), psl_key(labels[-1][0]), labels))
 
-		sorted_group = psl_sort(group)
 		if group != sorted_group:
 			warning('Incorrectly sorted group of domains')
-			print_list(group)
-			print_list(sorted_group)
+			print("  " + str(group))
+			print("  " + str(sorted_group))
+			print_psl(sorted_group)
 
 	finally:
 		del group[:]
 
 
 def lint_psl(infile):
-	"""Parses PSL file and extract strings and return code"""
+	"""Parses PSL file and performs syntax checking"""
 	global line, nline
 
 	PSL_FLAG_EXCEPTION = (1<<0)
@@ -109,12 +107,12 @@ def lint_psl(infile):
 
 		# empty line (end of sorted domain group)
 		if not line:
-			# check_order(group)
+			check_order(group)
 			continue
 
 		# check for section begin/end
 		if line[0:2] == "//":
-			# check_order(group)
+			check_order(group)
 
 			if section == 0:
 				if line == "// ===BEGIN ICANN DOMAINS===":
@@ -146,6 +144,8 @@ def lint_psl(infile):
 		if section == 0:
 			error('Rule outside of section')
 
+		group.append(list(reversed(line.split('.'))))
+
 		# decode UTF-8 input into unicode, needed only for python 2.x
 		if sys.version_info[0] < 3:
 			line = line.decode('utf-8')
@@ -174,10 +174,10 @@ def lint_psl(infile):
 		labels = line.split('.')
 
 		# collect reversed list of labels
-		if sys.version_info[0] < 3:
-			group.append(list(reversed(line.encode('utf-8').split('.'))))
-		else:
-			group.append(list(reversed(line.split('.'))))
+#		if sys.version_info[0] < 3:
+#			group.append(list(reversed(line.encode('utf-8').split('.'))))
+#		else:
+#			group.append(list(reversed(line.split('.'))))
 
 		for label in labels:
 			if not label:

From 811513f17e9412ba131b4447138f620392ebd645 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20R=C3=BChsen?= <tim.ruehsen@gmx.de>
Date: Fri, 12 Feb 2016 12:27:25 +0100
Subject: [PATCH 53/59] Print message and exit when no suffixes are found

---
 src/psl2c.c | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/psl2c.c b/src/psl2c.c
index 8531d40..2a4549c 100644
--- a/src/psl2c.c
+++ b/src/psl2c.c
@@ -256,6 +256,11 @@ int main(int argc, const char **argv)
 	if (!(psl = psl_load_file(argv[argpos])))
 		return 2;
 
+	if (!psl->suffixes || !psl->nsuffixes) {
+		fprintf(stderr, "Failed to load PSL. Please check content of '%s'.\n", argv[argpos]);
+		return 5;
+	}
+
 	/* look for ambiguous or double entries */
 /*	if (_check_psl(psl)) {
 		psl_free(psl);

From 2914afa8c7372b187f2c23d7d3b2a1da51a9c351 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20R=C3=BChsen?= <tim.ruehsen@gmx.de>
Date: Thu, 18 Feb 2016 16:40:06 +0100
Subject: [PATCH 54/59] New linter/ dir with pslint.py selftest

---
 .gitignore                        |  9 ++--
 Makefile.am                       |  3 ++
 {src => linter}/pslint.py         | 70 +++++++++++++++++++++++--------
 linter/pslint_selftest.sh         | 20 +++++++++
 linter/test_allowedchars.expected |  4 ++
 linter/test_allowedchars.input    | 15 +++++++
 linter/test_dots.expected         |  4 ++
 linter/test_dots.input            | 13 ++++++
 linter/test_duplicate.expected    |  6 +++
 linter/test_duplicate.input       | 26 ++++++++++++
 linter/test_exception.expected    |  6 +++
 linter/test_exception.input       | 23 ++++++++++
 linter/test_punycode.expected     |  3 ++
 linter/test_punycode.input        | 10 +++++
 linter/test_section1.expected     |  3 ++
 linter/test_section1.input        |  4 ++
 linter/test_section2.expected     |  2 +
 linter/test_section2.input        | 11 +++++
 linter/test_section3.expected     |  2 +
 linter/test_section3.input        | 11 +++++
 linter/test_section4.expected     |  3 ++
 linter/test_section4.input        |  8 ++++
 linter/test_spaces.expected       |  7 ++++
 linter/test_spaces.input          | 19 +++++++++
 linter/test_wildcard.expected     |  5 +++
 linter/test_wildcard.input        | 16 +++++++
 26 files changed, 282 insertions(+), 21 deletions(-)
 rename {src => linter}/pslint.py (78%)
 create mode 100755 linter/pslint_selftest.sh
 create mode 100644 linter/test_allowedchars.expected
 create mode 100644 linter/test_allowedchars.input
 create mode 100644 linter/test_dots.expected
 create mode 100644 linter/test_dots.input
 create mode 100644 linter/test_duplicate.expected
 create mode 100644 linter/test_duplicate.input
 create mode 100644 linter/test_exception.expected
 create mode 100644 linter/test_exception.input
 create mode 100644 linter/test_punycode.expected
 create mode 100644 linter/test_punycode.input
 create mode 100644 linter/test_section1.expected
 create mode 100644 linter/test_section1.input
 create mode 100644 linter/test_section2.expected
 create mode 100644 linter/test_section2.input
 create mode 100644 linter/test_section3.expected
 create mode 100644 linter/test_section3.input
 create mode 100644 linter/test_section4.expected
 create mode 100644 linter/test_section4.input
 create mode 100644 linter/test_spaces.expected
 create mode 100644 linter/test_spaces.input
 create mode 100644 linter/test_wildcard.expected
 create mode 100644 linter/test_wildcard.input

diff --git a/.gitignore b/.gitignore
index d49817a..3988824 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,8 +1,9 @@
-*.gz
-*.o
-*.lo
-*.la
 *.exe
+*.gz
+*.la
+*.lo
+*.log
+*.o
 *~
 */.deps
 */.libs
diff --git a/Makefile.am b/Makefile.am
index 3904754..482f2db 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -14,6 +14,9 @@ ACLOCAL_AMFLAGS = -I m4 ${ACLOCAL_FLAGS}
 pkgconfigdir = $(libdir)/pkgconfig
 pkgconfig_DATA = libpsl.pc
 
+clean-local:
+	rm -rf $(srcdir)/linter/log
+
 EXTRA_DIST = config.rpath LICENSE
 dist-hook:
 	mkdir -p $(distdir)/list/tests
diff --git a/src/pslint.py b/linter/pslint.py
similarity index 78%
rename from src/pslint.py
rename to linter/pslint.py
index 6148914..f8fc2d6 100755
--- a/src/pslint.py
+++ b/linter/pslint.py
@@ -27,18 +27,19 @@ import sys
 
 nline = 0
 line = ""
+orig_line = ""
 warnings = 0
 errors = 0
 skip_order_check = False
 
 def warning(msg):
-	global warnings, line, nline
-	print('%d: warning: %s%s' % (nline, msg, ": \'" + line + "\'" if line else ""))
+	global warnings, orig_line, nline
+	print('%d: warning: %s%s' % (nline, msg, ": \'" + orig_line + "\'" if orig_line else ""))
 	warnings += 1
 
 def error(msg):
-	global errors, line, nline
-	print('%d: error: %s%s' % (nline, msg, ": \'" + line + "\'" if line else ""))
+	global errors, orig_line, nline
+	print('%d: error: %s%s' % (nline, msg, ": \'" + orig_line + "\'" if orig_line else ""))
 	errors += 1
 #	skip_order_check = True
 
@@ -73,6 +74,7 @@ def check_order(group):
 			warning('Incorrectly sorted group of domains')
 			print("  " + str(group))
 			print("  " + str(sorted_group))
+                        print("Correct sorting would be:")
 			print_psl(sorted_group)
 
 	finally:
@@ -81,7 +83,7 @@ def check_order(group):
 
 def lint_psl(infile):
 	"""Parses PSL file and performs syntax checking"""
-	global line, nline
+	global orig_line, nline
 
 	PSL_FLAG_EXCEPTION = (1<<0)
 	PSL_FLAG_WILDCARD = (1<<1)
@@ -91,10 +93,12 @@ def lint_psl(infile):
 
 	line2number = {}
 	line2flag = {}
-	section = 0
 	group = []
+	section = 0
+	icann_sections = 0
+	private_sections = 0
 
-	lines = [line.strip('\r\n') for line in infile]
+	lines = [line.strip('\n') for line in infile]
 
 	for line in lines:
 		nline += 1
@@ -102,23 +106,28 @@ def lint_psl(infile):
 		# check for leadind/trailing whitespace
 		stripped = line.strip()
 		if stripped != line:
+			line = line.replace('\t','\\t')
+			line = line.replace('\r','^M')
 			warning('Leading/Trailing whitespace')
+		orig_line = line
 		line = stripped
 
 		# empty line (end of sorted domain group)
 		if not line:
-			check_order(group)
+			# check_order(group)
 			continue
 
 		# check for section begin/end
 		if line[0:2] == "//":
-			check_order(group)
+			# check_order(group)
 
 			if section == 0:
 				if line == "// ===BEGIN ICANN DOMAINS===":
 					section = PSL_FLAG_ICANN
+					icann_sections += 1
 				elif line == "// ===BEGIN PRIVATE DOMAINS===":
 					section = PSL_FLAG_PRIVATE
+					private_sections += 1
 				elif line[3:11] == "===BEGIN":
 					error('Unexpected begin of unknown section')
 				elif line[3:9] == "===END":
@@ -148,7 +157,11 @@ def lint_psl(infile):
 
 		# decode UTF-8 input into unicode, needed only for python 2.x
 		if sys.version_info[0] < 3:
-			line = line.decode('utf-8')
+			try:
+				line = line.decode('utf-8')
+			except UnicodeDecodeError:
+				error('Invalid UTF-8 character')
+				continue
 
 		# each rule must be lowercase (or more exactly: not uppercase and not titlecase)
 		if line != line.lower():
@@ -170,6 +183,7 @@ def lint_psl(infile):
 		# wildcard and exception must not combine
 		if flags & PSL_FLAG_WILDCARD and flags & PSL_FLAG_EXCEPTION:
 			error('Combination of wildcard and exception')
+			continue
 
 		labels = line.split('.')
 
@@ -179,18 +193,25 @@ def lint_psl(infile):
 #		else:
 #			group.append(list(reversed(line.split('.'))))
 
+		if flags & PSL_FLAG_EXCEPTION and len(labels) > 1:
+			domain = ".".join(str(label) for label in labels[1:])
+			if not domain in line2flag:
+				error('Exception without previous wildcard')
+			elif not line2flag[domain] & PSL_FLAG_WILDCARD:
+				error('Exception without previous wildcard')
+
 		for label in labels:
 			if not label:
-				 error('Leading/trailing or multiple dot')
-				 continue
+				error('Leading/trailing or multiple dot')
+				continue
 
 			if label[0:4] == 'xn--':
-				 error('Punycode found')
-				 continue
+				error('Punycode found')
+				continue
 
 			if '--' in label:
-				 error('Double minus found')
-				 continue
+				error('Double minus found')
+				continue
 
 			# allowed are a-z,0-9,- and unicode >= 128 (maybe that can be finetuned a bit !?)
 			for c in label:
@@ -208,11 +229,26 @@ def lint_psl(infile):
 			     !foo.bar + *.foo.bar
 			'''
 			error('Found doublette/ambiguity (previous line was %d)' % line2number[line])
-			continue
 
 		line2number[line] = nline
 		line2flag[line] = flags
 
+	orig_line = None
+
+	if section == PSL_FLAG_ICANN:
+		error('ICANN section not closed')
+	elif section == PSL_FLAG_PRIVATE:
+		error('PRIVATE section not closed')
+
+	if icann_sections < 1:
+		warning('No ICANN section found')
+	elif icann_sections > 1:
+		warning('%d ICANN sections found' % icann_sections)
+
+	if private_sections < 1:
+		warning('No PRIVATE section found')
+	elif private_sections > 1:
+		warning('%d PRIVATE sections found' % private_sections)
 
 def usage():
 	"""Prints the usage"""
diff --git a/linter/pslint_selftest.sh b/linter/pslint_selftest.sh
new file mode 100755
index 0000000..583e838
--- /dev/null
+++ b/linter/pslint_selftest.sh
@@ -0,0 +1,20 @@
+#!/bin/sh
+
+rc=0
+rm -rf log
+mkdir -p log
+
+for file in `ls *.input|cut -d'.' -f1`; do
+  echo -n "${file}: "
+  ./pslint.py ${file}.input >log/${file}.log 2>&1
+  diff ${file}.expected log/${file}.log >log/${file}.diff
+  if [ $? -eq 0 ]; then
+    echo OK
+    rm log/${file}.diff
+  else
+    echo FAILED
+    rc=1
+  fi
+done
+
+exit $rc
diff --git a/linter/test_allowedchars.expected b/linter/test_allowedchars.expected
new file mode 100644
index 0000000..58e0b69
--- /dev/null
+++ b/linter/test_allowedchars.expected
@@ -0,0 +1,4 @@
+10: error: Illegal character: 'a.exam#ple.com'
+11: error: Illegal character: 'b.exam ple.com'
+13: error: Invalid UTF-8 character: 'd.测è¯'
+15: warning: No PRIVATE section found
diff --git a/linter/test_allowedchars.input b/linter/test_allowedchars.input
new file mode 100644
index 0000000..f6d8778
--- /dev/null
+++ b/linter/test_allowedchars.input
@@ -0,0 +1,15 @@
+// test:
+// - label contains illegal character
+// - c. is valid UTF-8
+// - d. has invalid UTF-8 code for the TLD
+//
+// best viewed with 'LC_ALL=C vi <filename>'
+
+// ===BEGIN ICANN DOMAINS===
+
+a.exam#ple.com
+b.exam ple.com
+c.测试
+d.测è¯
+
+// ===END ICANN DOMAINS===
diff --git a/linter/test_dots.expected b/linter/test_dots.expected
new file mode 100644
index 0000000..1987840
--- /dev/null
+++ b/linter/test_dots.expected
@@ -0,0 +1,4 @@
+9: error: Leading/trailing or multiple dot: '.a.example.com'
+10: error: Leading/trailing or multiple dot: 'b.example.com.'
+11: error: Leading/trailing or multiple dot: 'c..example.com'
+13: warning: No PRIVATE section found
diff --git a/linter/test_dots.input b/linter/test_dots.input
new file mode 100644
index 0000000..3290441
--- /dev/null
+++ b/linter/test_dots.input
@@ -0,0 +1,13 @@
+// test:
+// - leading dot
+// - trailing dot
+// - consecutive dots
+
+// ===BEGIN ICANN DOMAINS===
+
+// example.com: https://www.iana.org/domains/reserved
+.a.example.com
+b.example.com.
+c..example.com
+
+// ===END ICANN DOMAINS===
diff --git a/linter/test_duplicate.expected b/linter/test_duplicate.expected
new file mode 100644
index 0000000..40a5e34
--- /dev/null
+++ b/linter/test_duplicate.expected
@@ -0,0 +1,6 @@
+9: error: Found doublette/ambiguity (previous line was 8): '*.com'
+13: error: Found doublette/ambiguity (previous line was 12): '!www.com'
+17: error: Found doublette/ambiguity (previous line was 16): '*.example.com'
+21: error: Found doublette/ambiguity (previous line was 20): 'example1.com'
+24: error: Found doublette/ambiguity (previous line was 17): 'example.com'
+26: warning: No PRIVATE section found
diff --git a/linter/test_duplicate.input b/linter/test_duplicate.input
new file mode 100644
index 0000000..48cd9ff
--- /dev/null
+++ b/linter/test_duplicate.input
@@ -0,0 +1,26 @@
+// test:
+// - valid wildcard usage
+// - invalid wildcard usage
+
+// ===BEGIN ICANN DOMAINS===
+
+// *.com implicitely includes .com
+com
+*.com
+
+// double exception
+!www.com
+!www.com
+
+// double wildcard
+*.example.com
+*.example.com
+
+// double plain rule
+example1.com
+example1.com
+
+// redundant/overlapping rule
+example.com
+
+// ===END ICANN DOMAINS===
diff --git a/linter/test_exception.expected b/linter/test_exception.expected
new file mode 100644
index 0000000..f656d6f
--- /dev/null
+++ b/linter/test_exception.expected
@@ -0,0 +1,6 @@
+17: error: Leading/trailing or multiple dot: '!.example.com'
+18: error: Illegal character: 'w!w.example.com'
+19: error: Found doublette/ambiguity (previous line was 12): '!www.example.com'
+20: error: Exception without previous wildcard: '!a.b.example.com'
+21: error: Exception without previous wildcard: '!a.c.example.com'
+23: warning: No PRIVATE section found
diff --git a/linter/test_exception.input b/linter/test_exception.input
new file mode 100644
index 0000000..96b1d7b
--- /dev/null
+++ b/linter/test_exception.input
@@ -0,0 +1,23 @@
+// test:
+// - valid exception
+// - invalid exceptions
+// - same exception twice
+// - exception without wildcard
+// - exception with prevailing '*' rule (!localhost)
+
+// ===BEGIN ICANN DOMAINS===
+
+// valid
+*.example.com
+!www.example.com
+!localhost
+c.example.com
+
+// invalid
+!.example.com
+w!w.example.com
+!www.example.com
+!a.b.example.com
+!a.c.example.com
+
+// ===END ICANN DOMAINS===
diff --git a/linter/test_punycode.expected b/linter/test_punycode.expected
new file mode 100644
index 0000000..1c4ab0e
--- /dev/null
+++ b/linter/test_punycode.expected
@@ -0,0 +1,3 @@
+7: error: Punycode found: 'a.xn--0zwm56d'
+8: error: Double minus found: 'a.ex--ample.com'
+10: warning: No PRIVATE section found
diff --git a/linter/test_punycode.input b/linter/test_punycode.input
new file mode 100644
index 0000000..be49210
--- /dev/null
+++ b/linter/test_punycode.input
@@ -0,0 +1,10 @@
+// test:
+// - label is punycode
+// - label has double minus
+
+// ===BEGIN ICANN DOMAINS===
+
+a.xn--0zwm56d
+a.ex--ample.com
+
+// ===END ICANN DOMAINS===
diff --git a/linter/test_section1.expected b/linter/test_section1.expected
new file mode 100644
index 0000000..d5eedfb
--- /dev/null
+++ b/linter/test_section1.expected
@@ -0,0 +1,3 @@
+4: error: Rule outside of section: 'example.com'
+4: warning: No ICANN section found
+4: warning: No PRIVATE section found
diff --git a/linter/test_section1.input b/linter/test_section1.input
new file mode 100644
index 0000000..25f6b66
--- /dev/null
+++ b/linter/test_section1.input
@@ -0,0 +1,4 @@
+// test:
+// - no section at all
+
+example.com
diff --git a/linter/test_section2.expected b/linter/test_section2.expected
new file mode 100644
index 0000000..a5eafb9
--- /dev/null
+++ b/linter/test_section2.expected
@@ -0,0 +1,2 @@
+11: warning: 2 ICANN sections found
+11: warning: No PRIVATE section found
diff --git a/linter/test_section2.input b/linter/test_section2.input
new file mode 100644
index 0000000..90f40a2
--- /dev/null
+++ b/linter/test_section2.input
@@ -0,0 +1,11 @@
+// test:
+// - two ICANN sections
+
+// ===BEGIN ICANN DOMAINS===
+
+example.com
+
+// ===END ICANN DOMAINS===
+
+// ===BEGIN ICANN DOMAINS===
+// ===END ICANN DOMAINS===
diff --git a/linter/test_section3.expected b/linter/test_section3.expected
new file mode 100644
index 0000000..fd28121
--- /dev/null
+++ b/linter/test_section3.expected
@@ -0,0 +1,2 @@
+11: warning: No ICANN section found
+11: warning: 2 PRIVATE sections found
diff --git a/linter/test_section3.input b/linter/test_section3.input
new file mode 100644
index 0000000..f3af185
--- /dev/null
+++ b/linter/test_section3.input
@@ -0,0 +1,11 @@
+// test:
+// - two PRIVATE sections
+
+// ===BEGIN PRIVATE DOMAINS===
+
+example.com
+
+// ===END PRIVATE DOMAINS===
+
+// ===BEGIN PRIVATE DOMAINS===
+// ===END PRIVATE DOMAINS===
diff --git a/linter/test_section4.expected b/linter/test_section4.expected
new file mode 100644
index 0000000..e8a0906
--- /dev/null
+++ b/linter/test_section4.expected
@@ -0,0 +1,3 @@
+8: error: Unexpected end of section: '// ===END PRIVATE DOMAINS==='
+8: error: ICANN section not closed
+8: warning: No PRIVATE section found
diff --git a/linter/test_section4.input b/linter/test_section4.input
new file mode 100644
index 0000000..98ca338
--- /dev/null
+++ b/linter/test_section4.input
@@ -0,0 +1,8 @@
+// test:
+// - ICANN section improperly closed
+
+// ===BEGIN ICANN DOMAINS===
+
+example.com
+
+// ===END PRIVATE DOMAINS===
diff --git a/linter/test_spaces.expected b/linter/test_spaces.expected
new file mode 100644
index 0000000..63650ed
--- /dev/null
+++ b/linter/test_spaces.expected
@@ -0,0 +1,7 @@
+12: warning: Leading/Trailing whitespace: '// example.com: https://www.iana.org/domains/reserved'
+13: warning: Leading/Trailing whitespace: ' a.example.com'
+14: warning: Leading/Trailing whitespace: 'b.example.com '
+15: warning: Leading/Trailing whitespace: '\tc.example.com'
+16: warning: Leading/Trailing whitespace: 'd.example.com\t'
+17: warning: Leading/Trailing whitespace: 'e.example.com^M'
+19: warning: No PRIVATE section found
diff --git a/linter/test_spaces.input b/linter/test_spaces.input
new file mode 100644
index 0000000..f84f866
--- /dev/null
+++ b/linter/test_spaces.input
@@ -0,0 +1,19 @@
+// test:
+// - leading space
+// - trailing space, empty line with spaces
+// - leading tab
+// - trailing tab
+// - line ends with CRLF
+// - empty line with spaces
+
+// ===BEGIN ICANN DOMAINS===
+
+// example.com: https://www.iana.org/domains/reserved
+ a.example.com
+b.example.com 
+	c.example.com
+d.example.com	
+e.example.com
+  
+
+// ===END ICANN DOMAINS===
diff --git a/linter/test_wildcard.expected b/linter/test_wildcard.expected
new file mode 100644
index 0000000..5457094
--- /dev/null
+++ b/linter/test_wildcard.expected
@@ -0,0 +1,5 @@
+11: error: Illegal character: '**.com'
+12: error: Illegal character: 'a*.com'
+13: error: Illegal character: 'b.*.com'
+14: error: Illegal character: 'a.b.*'
+16: warning: No PRIVATE section found
diff --git a/linter/test_wildcard.input b/linter/test_wildcard.input
new file mode 100644
index 0000000..1bfb05a
--- /dev/null
+++ b/linter/test_wildcard.input
@@ -0,0 +1,16 @@
+// test:
+// - valid wildcard usage
+// - invalid wildcard usage
+
+// ===BEGIN ICANN DOMAINS===
+
+// valid
+*.com
+
+// invalid
+**.com
+a*.com
+b.*.com
+a.b.*
+
+// ===END ICANN DOMAINS===

From 4798ee88a2ce3d1049d8fe2ce286a34905899de5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20R=C3=BChsen?= <tim.ruehsen@gmx.de>
Date: Sat, 20 Feb 2016 15:06:48 +0100
Subject: [PATCH 55/59] Add test suite support for list/tests/tests.txt

---
 tests/test-registrable-domain.c | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/tests/test-registrable-domain.c b/tests/test-registrable-domain.c
index c9c3604..6a227e1 100644
--- a/tests/test-registrable-domain.c
+++ b/tests/test-registrable-domain.c
@@ -130,6 +130,11 @@ static void test_psl(void)
 				} else if (sscanf(p, "checkPublicSuffix ( null , null ) %1[;]", semicolon) == 1) {
 					d_is_null = 1;
 					er_is_null = 1;
+				} else if (sscanf(p, "%127s %127s", domain, expected_regdom) == 2) {
+					if (!strcmp(domain, "null"))
+						d_is_null = 1;
+					if (!strcmp(expected_regdom, "null"))
+						er_is_null = 1;
 				} else {
 					failed++;
 					printf("Malformed line from '" PSL_TESTFILE "': %s", buf);

From cb3fc3b502353816fad98f60915c30e8911a046a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20R=C3=BChsen?= <tim.ruehsen@gmx.de>
Date: Sat, 20 Feb 2016 23:05:34 +0100
Subject: [PATCH 56/59] Travis CI only for branches develop and master

---
 .travis.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.travis.yml b/.travis.yml
index 3a0823c..5b40aa7 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -6,6 +6,11 @@ compiler:
   - gcc
   - clang
 
+branches:
+    only:
+        - master
+        - develop
+
 env:
     - RUNTIME=libicu
     - RUNTIME=libidn2

From efdd800cb346466c24cf297b214eda3a4582da81 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20R=C3=BChsen?= <tim.ruehsen@gmx.de>
Date: Sat, 20 Feb 2016 22:20:30 +0100
Subject: [PATCH 57/59] Use tests.txt as PSL test file by default

---
 configure.ac | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/configure.ac b/configure.ac
index a14f4ec..4179c68 100644
--- a/configure.ac
+++ b/configure.ac
@@ -255,7 +255,7 @@ AC_SUBST(PSL_FILE)
 AC_ARG_WITH(psl-testfile,
   AC_HELP_STRING([--with-psl-testfile=[PATH]], [path to PSL test file]),
   PSL_TESTFILE=$withval,
-  PSL_TESTFILE="\$(top_srcdir)/list/tests/test_psl.txt")
+  PSL_TESTFILE="\$(top_srcdir)/list/tests/tests.txt")
 AC_SUBST(PSL_TESTFILE)
 
 # check for alloca / alloca.h

From f097290c90e30b873d7ef73b9c9d194081f72afe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20R=C3=BChsen?= <tim.ruehsen@gmx.de>
Date: Wed, 2 Mar 2016 11:45:28 +0100
Subject: [PATCH 58/59] Remove python linter

It has been merged into https://github.com/publicsuffix/list
and can be found in list/linter.
---
 Makefile.am                       |   3 -
 linter/pslint.py                  | 272 ------------------------------
 linter/pslint_selftest.sh         |  20 ---
 linter/test_allowedchars.expected |   4 -
 linter/test_allowedchars.input    |  15 --
 linter/test_dots.expected         |   4 -
 linter/test_dots.input            |  13 --
 linter/test_duplicate.expected    |   6 -
 linter/test_duplicate.input       |  26 ---
 linter/test_exception.expected    |   6 -
 linter/test_exception.input       |  23 ---
 linter/test_punycode.expected     |   3 -
 linter/test_punycode.input        |  10 --
 linter/test_section1.expected     |   3 -
 linter/test_section1.input        |   4 -
 linter/test_section2.expected     |   2 -
 linter/test_section2.input        |  11 --
 linter/test_section3.expected     |   2 -
 linter/test_section3.input        |  11 --
 linter/test_section4.expected     |   3 -
 linter/test_section4.input        |   8 -
 linter/test_spaces.expected       |   7 -
 linter/test_spaces.input          |  19 ---
 linter/test_wildcard.expected     |   5 -
 linter/test_wildcard.input        |  16 --
 list                              |   2 +-
 26 files changed, 1 insertion(+), 497 deletions(-)
 delete mode 100755 linter/pslint.py
 delete mode 100755 linter/pslint_selftest.sh
 delete mode 100644 linter/test_allowedchars.expected
 delete mode 100644 linter/test_allowedchars.input
 delete mode 100644 linter/test_dots.expected
 delete mode 100644 linter/test_dots.input
 delete mode 100644 linter/test_duplicate.expected
 delete mode 100644 linter/test_duplicate.input
 delete mode 100644 linter/test_exception.expected
 delete mode 100644 linter/test_exception.input
 delete mode 100644 linter/test_punycode.expected
 delete mode 100644 linter/test_punycode.input
 delete mode 100644 linter/test_section1.expected
 delete mode 100644 linter/test_section1.input
 delete mode 100644 linter/test_section2.expected
 delete mode 100644 linter/test_section2.input
 delete mode 100644 linter/test_section3.expected
 delete mode 100644 linter/test_section3.input
 delete mode 100644 linter/test_section4.expected
 delete mode 100644 linter/test_section4.input
 delete mode 100644 linter/test_spaces.expected
 delete mode 100644 linter/test_spaces.input
 delete mode 100644 linter/test_wildcard.expected
 delete mode 100644 linter/test_wildcard.input

diff --git a/Makefile.am b/Makefile.am
index 482f2db..3904754 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -14,9 +14,6 @@ ACLOCAL_AMFLAGS = -I m4 ${ACLOCAL_FLAGS}
 pkgconfigdir = $(libdir)/pkgconfig
 pkgconfig_DATA = libpsl.pc
 
-clean-local:
-	rm -rf $(srcdir)/linter/log
-
 EXTRA_DIST = config.rpath LICENSE
 dist-hook:
 	mkdir -p $(distdir)/list/tests
diff --git a/linter/pslint.py b/linter/pslint.py
deleted file mode 100755
index f8fc2d6..0000000
--- a/linter/pslint.py
+++ /dev/null
@@ -1,272 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-#
-#
-# PSL linter written in python
-#
-# Copyright 2016 Tim Rühsen (tim dot ruehsen at gmx dot de). All rights reserved.
-#
-# Permission is hereby granted, free of charge, to any person obtaining a
-# copy of this software and associated documentation files (the "Software"),
-# to deal in the Software without restriction, including without limitation
-# the rights to use, copy, modify, merge, publish, distribute, sublicense,
-# and/or sell copies of the Software, and to permit persons to whom the
-# Software is furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-# DEALINGS IN THE SOFTWARE.
-
-import sys
-
-nline = 0
-line = ""
-orig_line = ""
-warnings = 0
-errors = 0
-skip_order_check = False
-
-def warning(msg):
-	global warnings, orig_line, nline
-	print('%d: warning: %s%s' % (nline, msg, ": \'" + orig_line + "\'" if orig_line else ""))
-	warnings += 1
-
-def error(msg):
-	global errors, orig_line, nline
-	print('%d: error: %s%s' % (nline, msg, ": \'" + orig_line + "\'" if orig_line else ""))
-	errors += 1
-#	skip_order_check = True
-
-def print_psl(list):
-	for domain in list:
-		print(".".join(str(label) for label in reversed(domain)))
-
-def psl_key(s):
-	if s[0] == '*':
-		return 0
-	if s[0] == '!':
-		return 1
-	return 2
-
-def check_order(group):
-	"""Check the correct order of a domain group"""
-	global skip_order_check
-
-	try:
-		if skip_order_check or len(group) < 2:
-			skip_order_check = False
-			return
-
-		# check if the TLD is the identical within the group
-		if any(group[0][0] != labels[0] for labels in group):
-			warning('Domain group TLD is not consistent')
-
-		# sort by # of labels, label-by-label (labels are in reversed order)
-		sorted_group = sorted(group, key = lambda labels: (len(labels), psl_key(labels[-1][0]), labels))
-
-		if group != sorted_group:
-			warning('Incorrectly sorted group of domains')
-			print("  " + str(group))
-			print("  " + str(sorted_group))
-                        print("Correct sorting would be:")
-			print_psl(sorted_group)
-
-	finally:
-		del group[:]
-
-
-def lint_psl(infile):
-	"""Parses PSL file and performs syntax checking"""
-	global orig_line, nline
-
-	PSL_FLAG_EXCEPTION = (1<<0)
-	PSL_FLAG_WILDCARD = (1<<1)
-	PSL_FLAG_ICANN = (1<<2) # entry of ICANN section
-	PSL_FLAG_PRIVATE = (1<<3) # entry of PRIVATE section
-	PSL_FLAG_PLAIN = (1<<4) #just used for PSL syntax checking
-
-	line2number = {}
-	line2flag = {}
-	group = []
-	section = 0
-	icann_sections = 0
-	private_sections = 0
-
-	lines = [line.strip('\n') for line in infile]
-
-	for line in lines:
-		nline += 1
-
-		# check for leadind/trailing whitespace
-		stripped = line.strip()
-		if stripped != line:
-			line = line.replace('\t','\\t')
-			line = line.replace('\r','^M')
-			warning('Leading/Trailing whitespace')
-		orig_line = line
-		line = stripped
-
-		# empty line (end of sorted domain group)
-		if not line:
-			# check_order(group)
-			continue
-
-		# check for section begin/end
-		if line[0:2] == "//":
-			# check_order(group)
-
-			if section == 0:
-				if line == "// ===BEGIN ICANN DOMAINS===":
-					section = PSL_FLAG_ICANN
-					icann_sections += 1
-				elif line == "// ===BEGIN PRIVATE DOMAINS===":
-					section = PSL_FLAG_PRIVATE
-					private_sections += 1
-				elif line[3:11] == "===BEGIN":
-					error('Unexpected begin of unknown section')
-				elif line[3:9] == "===END":
-					error('End of section without previous begin')
-			elif section == PSL_FLAG_ICANN:
-				if line == "// ===END ICANN DOMAINS===":
-					section = 0
-				elif line[3:11] == "===BEGIN":
-					error('Unexpected begin of section: ')
-				elif line[3:9] == "===END":
-					error('Unexpected end of section')
-			elif section == PSL_FLAG_PRIVATE:
-				if line == "// ===END PRIVATE DOMAINS===":
-					section = 0
-				elif line[3:11] == "===BEGIN":
-					error('Unexpected begin of section')
-				elif line[3:9] == "===END":
-					error('Unexpected end of section')
-
-			continue # processing of comments ends here
-
-		# No rule must be outside of a section
-		if section == 0:
-			error('Rule outside of section')
-
-		group.append(list(reversed(line.split('.'))))
-
-		# decode UTF-8 input into unicode, needed only for python 2.x
-		if sys.version_info[0] < 3:
-			try:
-				line = line.decode('utf-8')
-			except UnicodeDecodeError:
-				error('Invalid UTF-8 character')
-				continue
-
-		# each rule must be lowercase (or more exactly: not uppercase and not titlecase)
-		if line != line.lower():
-			error('Rule must be lowercase')
-
-		# strip leading wildcards
-		flags = section
-		# while line[0:2] == '*.':
-		if line[0:2] == '*.':
-			flags |= PSL_FLAG_WILDCARD
-			line = line[2:]
-
-		if line[0] == '!':
-			flags |= PSL_FLAG_EXCEPTION
-			line = line[1:]
-		else:
-			flags |= PSL_FLAG_PLAIN
-
-		# wildcard and exception must not combine
-		if flags & PSL_FLAG_WILDCARD and flags & PSL_FLAG_EXCEPTION:
-			error('Combination of wildcard and exception')
-			continue
-
-		labels = line.split('.')
-
-		# collect reversed list of labels
-#		if sys.version_info[0] < 3:
-#			group.append(list(reversed(line.encode('utf-8').split('.'))))
-#		else:
-#			group.append(list(reversed(line.split('.'))))
-
-		if flags & PSL_FLAG_EXCEPTION and len(labels) > 1:
-			domain = ".".join(str(label) for label in labels[1:])
-			if not domain in line2flag:
-				error('Exception without previous wildcard')
-			elif not line2flag[domain] & PSL_FLAG_WILDCARD:
-				error('Exception without previous wildcard')
-
-		for label in labels:
-			if not label:
-				error('Leading/trailing or multiple dot')
-				continue
-
-			if label[0:4] == 'xn--':
-				error('Punycode found')
-				continue
-
-			if '--' in label:
-				error('Double minus found')
-				continue
-
-			# allowed are a-z,0-9,- and unicode >= 128 (maybe that can be finetuned a bit !?)
-			for c in label:
-				if not c.isalnum() and c != '-' and ord(c) < 128:
-					error('Illegal character')
-					break
-
-		if line in line2flag:
-			'''Found existing entry:
-			   Combination of exception and plain rule is contradictionary
-			     !foo.bar + foo.bar
-			   Doublette, since *.foo.bar implies foo.bar:
-			      foo.bar + *.foo.bar
-			   Allowed:
-			     !foo.bar + *.foo.bar
-			'''
-			error('Found doublette/ambiguity (previous line was %d)' % line2number[line])
-
-		line2number[line] = nline
-		line2flag[line] = flags
-
-	orig_line = None
-
-	if section == PSL_FLAG_ICANN:
-		error('ICANN section not closed')
-	elif section == PSL_FLAG_PRIVATE:
-		error('PRIVATE section not closed')
-
-	if icann_sections < 1:
-		warning('No ICANN section found')
-	elif icann_sections > 1:
-		warning('%d ICANN sections found' % icann_sections)
-
-	if private_sections < 1:
-		warning('No PRIVATE section found')
-	elif private_sections > 1:
-		warning('%d PRIVATE sections found' % private_sections)
-
-def usage():
-	"""Prints the usage"""
-	print('usage: %s PSLfile' % sys.argv[0])
-	print('or     %s -        # To read PSL from STDIN' % sys.argv[0])
-	exit(1)
-
-
-def main():
-	"""Check syntax of a PSL file"""
-	if len(sys.argv) < 2:
-		usage()
-
-	with sys.stdin if sys.argv[-1] == '-' else open(sys.argv[-1], 'r') as infile:
-		lint_psl(infile)
-
-	return errors != 0
-
-
-if __name__ == '__main__':
-	sys.exit(main())
diff --git a/linter/pslint_selftest.sh b/linter/pslint_selftest.sh
deleted file mode 100755
index 583e838..0000000
--- a/linter/pslint_selftest.sh
+++ /dev/null
@@ -1,20 +0,0 @@
-#!/bin/sh
-
-rc=0
-rm -rf log
-mkdir -p log
-
-for file in `ls *.input|cut -d'.' -f1`; do
-  echo -n "${file}: "
-  ./pslint.py ${file}.input >log/${file}.log 2>&1
-  diff ${file}.expected log/${file}.log >log/${file}.diff
-  if [ $? -eq 0 ]; then
-    echo OK
-    rm log/${file}.diff
-  else
-    echo FAILED
-    rc=1
-  fi
-done
-
-exit $rc
diff --git a/linter/test_allowedchars.expected b/linter/test_allowedchars.expected
deleted file mode 100644
index 58e0b69..0000000
--- a/linter/test_allowedchars.expected
+++ /dev/null
@@ -1,4 +0,0 @@
-10: error: Illegal character: 'a.exam#ple.com'
-11: error: Illegal character: 'b.exam ple.com'
-13: error: Invalid UTF-8 character: 'd.测è¯'
-15: warning: No PRIVATE section found
diff --git a/linter/test_allowedchars.input b/linter/test_allowedchars.input
deleted file mode 100644
index f6d8778..0000000
--- a/linter/test_allowedchars.input
+++ /dev/null
@@ -1,15 +0,0 @@
-// test:
-// - label contains illegal character
-// - c. is valid UTF-8
-// - d. has invalid UTF-8 code for the TLD
-//
-// best viewed with 'LC_ALL=C vi <filename>'
-
-// ===BEGIN ICANN DOMAINS===
-
-a.exam#ple.com
-b.exam ple.com
-c.测试
-d.测è¯
-
-// ===END ICANN DOMAINS===
diff --git a/linter/test_dots.expected b/linter/test_dots.expected
deleted file mode 100644
index 1987840..0000000
--- a/linter/test_dots.expected
+++ /dev/null
@@ -1,4 +0,0 @@
-9: error: Leading/trailing or multiple dot: '.a.example.com'
-10: error: Leading/trailing or multiple dot: 'b.example.com.'
-11: error: Leading/trailing or multiple dot: 'c..example.com'
-13: warning: No PRIVATE section found
diff --git a/linter/test_dots.input b/linter/test_dots.input
deleted file mode 100644
index 3290441..0000000
--- a/linter/test_dots.input
+++ /dev/null
@@ -1,13 +0,0 @@
-// test:
-// - leading dot
-// - trailing dot
-// - consecutive dots
-
-// ===BEGIN ICANN DOMAINS===
-
-// example.com: https://www.iana.org/domains/reserved
-.a.example.com
-b.example.com.
-c..example.com
-
-// ===END ICANN DOMAINS===
diff --git a/linter/test_duplicate.expected b/linter/test_duplicate.expected
deleted file mode 100644
index 40a5e34..0000000
--- a/linter/test_duplicate.expected
+++ /dev/null
@@ -1,6 +0,0 @@
-9: error: Found doublette/ambiguity (previous line was 8): '*.com'
-13: error: Found doublette/ambiguity (previous line was 12): '!www.com'
-17: error: Found doublette/ambiguity (previous line was 16): '*.example.com'
-21: error: Found doublette/ambiguity (previous line was 20): 'example1.com'
-24: error: Found doublette/ambiguity (previous line was 17): 'example.com'
-26: warning: No PRIVATE section found
diff --git a/linter/test_duplicate.input b/linter/test_duplicate.input
deleted file mode 100644
index 48cd9ff..0000000
--- a/linter/test_duplicate.input
+++ /dev/null
@@ -1,26 +0,0 @@
-// test:
-// - valid wildcard usage
-// - invalid wildcard usage
-
-// ===BEGIN ICANN DOMAINS===
-
-// *.com implicitely includes .com
-com
-*.com
-
-// double exception
-!www.com
-!www.com
-
-// double wildcard
-*.example.com
-*.example.com
-
-// double plain rule
-example1.com
-example1.com
-
-// redundant/overlapping rule
-example.com
-
-// ===END ICANN DOMAINS===
diff --git a/linter/test_exception.expected b/linter/test_exception.expected
deleted file mode 100644
index f656d6f..0000000
--- a/linter/test_exception.expected
+++ /dev/null
@@ -1,6 +0,0 @@
-17: error: Leading/trailing or multiple dot: '!.example.com'
-18: error: Illegal character: 'w!w.example.com'
-19: error: Found doublette/ambiguity (previous line was 12): '!www.example.com'
-20: error: Exception without previous wildcard: '!a.b.example.com'
-21: error: Exception without previous wildcard: '!a.c.example.com'
-23: warning: No PRIVATE section found
diff --git a/linter/test_exception.input b/linter/test_exception.input
deleted file mode 100644
index 96b1d7b..0000000
--- a/linter/test_exception.input
+++ /dev/null
@@ -1,23 +0,0 @@
-// test:
-// - valid exception
-// - invalid exceptions
-// - same exception twice
-// - exception without wildcard
-// - exception with prevailing '*' rule (!localhost)
-
-// ===BEGIN ICANN DOMAINS===
-
-// valid
-*.example.com
-!www.example.com
-!localhost
-c.example.com
-
-// invalid
-!.example.com
-w!w.example.com
-!www.example.com
-!a.b.example.com
-!a.c.example.com
-
-// ===END ICANN DOMAINS===
diff --git a/linter/test_punycode.expected b/linter/test_punycode.expected
deleted file mode 100644
index 1c4ab0e..0000000
--- a/linter/test_punycode.expected
+++ /dev/null
@@ -1,3 +0,0 @@
-7: error: Punycode found: 'a.xn--0zwm56d'
-8: error: Double minus found: 'a.ex--ample.com'
-10: warning: No PRIVATE section found
diff --git a/linter/test_punycode.input b/linter/test_punycode.input
deleted file mode 100644
index be49210..0000000
--- a/linter/test_punycode.input
+++ /dev/null
@@ -1,10 +0,0 @@
-// test:
-// - label is punycode
-// - label has double minus
-
-// ===BEGIN ICANN DOMAINS===
-
-a.xn--0zwm56d
-a.ex--ample.com
-
-// ===END ICANN DOMAINS===
diff --git a/linter/test_section1.expected b/linter/test_section1.expected
deleted file mode 100644
index d5eedfb..0000000
--- a/linter/test_section1.expected
+++ /dev/null
@@ -1,3 +0,0 @@
-4: error: Rule outside of section: 'example.com'
-4: warning: No ICANN section found
-4: warning: No PRIVATE section found
diff --git a/linter/test_section1.input b/linter/test_section1.input
deleted file mode 100644
index 25f6b66..0000000
--- a/linter/test_section1.input
+++ /dev/null
@@ -1,4 +0,0 @@
-// test:
-// - no section at all
-
-example.com
diff --git a/linter/test_section2.expected b/linter/test_section2.expected
deleted file mode 100644
index a5eafb9..0000000
--- a/linter/test_section2.expected
+++ /dev/null
@@ -1,2 +0,0 @@
-11: warning: 2 ICANN sections found
-11: warning: No PRIVATE section found
diff --git a/linter/test_section2.input b/linter/test_section2.input
deleted file mode 100644
index 90f40a2..0000000
--- a/linter/test_section2.input
+++ /dev/null
@@ -1,11 +0,0 @@
-// test:
-// - two ICANN sections
-
-// ===BEGIN ICANN DOMAINS===
-
-example.com
-
-// ===END ICANN DOMAINS===
-
-// ===BEGIN ICANN DOMAINS===
-// ===END ICANN DOMAINS===
diff --git a/linter/test_section3.expected b/linter/test_section3.expected
deleted file mode 100644
index fd28121..0000000
--- a/linter/test_section3.expected
+++ /dev/null
@@ -1,2 +0,0 @@
-11: warning: No ICANN section found
-11: warning: 2 PRIVATE sections found
diff --git a/linter/test_section3.input b/linter/test_section3.input
deleted file mode 100644
index f3af185..0000000
--- a/linter/test_section3.input
+++ /dev/null
@@ -1,11 +0,0 @@
-// test:
-// - two PRIVATE sections
-
-// ===BEGIN PRIVATE DOMAINS===
-
-example.com
-
-// ===END PRIVATE DOMAINS===
-
-// ===BEGIN PRIVATE DOMAINS===
-// ===END PRIVATE DOMAINS===
diff --git a/linter/test_section4.expected b/linter/test_section4.expected
deleted file mode 100644
index e8a0906..0000000
--- a/linter/test_section4.expected
+++ /dev/null
@@ -1,3 +0,0 @@
-8: error: Unexpected end of section: '// ===END PRIVATE DOMAINS==='
-8: error: ICANN section not closed
-8: warning: No PRIVATE section found
diff --git a/linter/test_section4.input b/linter/test_section4.input
deleted file mode 100644
index 98ca338..0000000
--- a/linter/test_section4.input
+++ /dev/null
@@ -1,8 +0,0 @@
-// test:
-// - ICANN section improperly closed
-
-// ===BEGIN ICANN DOMAINS===
-
-example.com
-
-// ===END PRIVATE DOMAINS===
diff --git a/linter/test_spaces.expected b/linter/test_spaces.expected
deleted file mode 100644
index 63650ed..0000000
--- a/linter/test_spaces.expected
+++ /dev/null
@@ -1,7 +0,0 @@
-12: warning: Leading/Trailing whitespace: '// example.com: https://www.iana.org/domains/reserved'
-13: warning: Leading/Trailing whitespace: ' a.example.com'
-14: warning: Leading/Trailing whitespace: 'b.example.com '
-15: warning: Leading/Trailing whitespace: '\tc.example.com'
-16: warning: Leading/Trailing whitespace: 'd.example.com\t'
-17: warning: Leading/Trailing whitespace: 'e.example.com^M'
-19: warning: No PRIVATE section found
diff --git a/linter/test_spaces.input b/linter/test_spaces.input
deleted file mode 100644
index f84f866..0000000
--- a/linter/test_spaces.input
+++ /dev/null
@@ -1,19 +0,0 @@
-// test:
-// - leading space
-// - trailing space, empty line with spaces
-// - leading tab
-// - trailing tab
-// - line ends with CRLF
-// - empty line with spaces
-
-// ===BEGIN ICANN DOMAINS===
-
-// example.com: https://www.iana.org/domains/reserved
- a.example.com
-b.example.com 
-	c.example.com
-d.example.com	
-e.example.com
-  
-
-// ===END ICANN DOMAINS===
diff --git a/linter/test_wildcard.expected b/linter/test_wildcard.expected
deleted file mode 100644
index 5457094..0000000
--- a/linter/test_wildcard.expected
+++ /dev/null
@@ -1,5 +0,0 @@
-11: error: Illegal character: '**.com'
-12: error: Illegal character: 'a*.com'
-13: error: Illegal character: 'b.*.com'
-14: error: Illegal character: 'a.b.*'
-16: warning: No PRIVATE section found
diff --git a/linter/test_wildcard.input b/linter/test_wildcard.input
deleted file mode 100644
index 1bfb05a..0000000
--- a/linter/test_wildcard.input
+++ /dev/null
@@ -1,16 +0,0 @@
-// test:
-// - valid wildcard usage
-// - invalid wildcard usage
-
-// ===BEGIN ICANN DOMAINS===
-
-// valid
-*.com
-
-// invalid
-**.com
-a*.com
-b.*.com
-a.b.*
-
-// ===END ICANN DOMAINS===
diff --git a/list b/list
index 1f3ad51..e2f2f4b 160000
--- a/list
+++ b/list
@@ -1 +1 @@
-Subproject commit 1f3ad51171235aafe423435606e869f0161582e4
+Subproject commit e2f2f4bfe2ae57651afb7268bb9a0b53da5eb8cf

From a6ed60e57ffc54ca7bc9880e8eec7b68de4ca770 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20R=C3=BChsen?= <tim.ruehsen@gmx.de>
Date: Wed, 2 Mar 2016 12:25:35 +0100
Subject: [PATCH 59/59] Release v0.13.0

---
 NEWS         | 6 ++++++
 configure.ac | 2 +-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/NEWS b/NEWS
index 57c9b95..565f5cc 100644
--- a/NEWS
+++ b/NEWS
@@ -1,5 +1,11 @@
 Copyright (C) 2014-2016 Tim Rühsen
 
+02.03.2016 Release V0.13.0
+  * Use tests.txt as PSL test file by default
+  * Slightly shorter DAFSA array when sorting input
+  * Check for python 2.7+ in configure.ac
+  * Fix python3 incompatibilities in make_dafsa.py
+
 02.01.2016 Release V0.12.0
   * Load DAFSA binaries via psl_load_file() via auto-detection
   * Add more tests
diff --git a/configure.ac b/configure.ac
index 4179c68..1cb1828 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1,5 +1,5 @@
 
-AC_INIT([libpsl], [0.12.0], [tim.ruehsen@gmx.de], [libpsl], [http://github.com/rockdaboot/libpsl])
+AC_INIT([libpsl], [0.13.0], [tim.ruehsen@gmx.de], [libpsl], [http://github.com/rockdaboot/libpsl])
 AC_PREREQ([2.59])
 AM_INIT_AUTOMAKE([1.10 -Wall no-define foreign])