Release v0.16.0

-----BEGIN PGP SIGNATURE-----
 
 iQIzBAABCgAdFiEEHLJ9vJhhSy1YQWRtCDAttqJnBCgFAlhMNkYACgkQCDAttqJn
 BCgO9w/7BopUogD/t2vDPm3CvTdlgJHvxlkYfG74DU/Qoxmh8XW2gfshsEOlY7TM
 PAew4V2R+JkKH1gJXyFdLofWF/pFDrzEoUG+ymFBNJ8Y8u++Hf4/6JagH706tarL
 zjDCEF6U4HXF6JHr9hvXbdc7g98Qo0+Ymro2O/ruNbZbP01unzNFIj3pa6zhvMeB
 rp0iir8i+kjMrUpnXkry8KblQ5UY9m/6DTCGOkmnKb9q+v1eHNWCCCnenlqQPsJ6
 VR5rNN38UpUlqDVsifaVSxJWnRo5CH57sZxXGQjZqsxhy0/vWq9kBNwNkDZbb9E7
 o7Dhu/WEE/5tQQLyJKVmWilVm1MEYvTmkNyW067+f9OGwMYuFudcw3PaFh6V/dou
 5Y1KSTQQBuaA6q9SmTw8C57x6VHuBRmJ2qI4RR+ruvqCjW8twOI2izQ/0MF9TWVG
 UQB+iJ6OxSQjvTtA1LdvI4/K038OIgtpV12QYGfFm9iMfsGylOQUJhG0QpDZb298
 +aikwnXdMqSqSqp17XFFE49xE+yrRGUpv3cYLThc2GWULd6mI4b5oikRE6eC5FOB
 jZ65TJu/Yp6lVJmb+kj4JcEnJlsEt7CMDp0xCkCYrWFwGYZgPhCLVPrEDFNAjf5F
 sAdHTR6h8OM6KdpPGOoyTCFXpFbQRkZTk5NeGZxi8okxd8hi5q4=
 =w6xS
 -----END PGP SIGNATURE-----

Merge tag 'libpsl-0.16.0' into debian

Release v0.16.0

Makefile.am

@@ -33,7 +33,6 @@ check-coverage:
 	lcov --capture --initial --directory src --output-file libpsl.info
 	$(MAKE) check
 	lcov --capture --directory src --output-file libpsl.info
-	lcov --remove libpsl.info 'src/psl2c.c' -o libpsl.info
 	genhtml --prefix . libpsl.info --legend --title "libpsl" --output-directory=lcov
 
 check-coverage-libidn:

NEWS

@@ -1,5 +1,13 @@
 Copyright (C) 2014-2016 Tim Rühsen
 
+16.12.2016 Release V0.16.0
+  * Add functions psl_latest() and psl_dist_filename()
+  * Do not taint out variable on error in psl_str_to_utf8lower()
+  * Replace psl2c by psl-make-dafsa
+  * Add missing includes for OpenBSD
+  * Fix typos
+  * Update copyright year
+
 14.11.2016 Release V0.15.0
   * Python3 compatibility for psl-make-dafsa
   * Support for UTF-8 in DAFSA data

README.md

@@ -30,9 +30,9 @@ Libpsl...
 - finds the shortest private part of a given domain
 - works with international domains (UTF-8 and IDNA2008 Punycode)
 - is thread-safe
-- handles IDNA2008 UTS#46 (libicu is used by psl2c if installed)
+- handles IDNA2008 UTS#46 (if libicu is available)
 
-Find more information about the Publix Suffix List [here](https://publicsuffix.org/).
+Find more information about the Public Suffix List [here](https://publicsuffix.org/).
 
 Download the Public Suffix List [here](https://hg.mozilla.org/mozilla-central/raw-file/tip/netwerk/dns/effective_tld_names.dat).
 
@@ -86,7 +86,7 @@ representation of strings. Here we use it to reduce the whole PSL to about 32k i
 
 Generate `psl.dafsa` from `list/public_suffix_list.dat`
 
-	$ src/psl-make-dafsa --output-format=binary --input-format=psl list/public_suffix_list.dat psl.dafsa
+	$ src/psl-make-dafsa --output-format=binary list/public_suffix_list.dat psl.dafsa
 
 Test the result (example)
 

configure.ac

@@ -1,5 +1,5 @@
 
-AC_INIT([libpsl], [0.15.0], [tim.ruehsen@gmx.de], [libpsl], [https://github.com/rockdaboot/libpsl])
+AC_INIT([libpsl], [0.16.0], [tim.ruehsen@gmx.de], [libpsl], [https://github.com/rockdaboot/libpsl])
 AC_PREREQ([2.59])
 AM_INIT_AUTOMAKE([1.10 no-define foreign])
 
@@ -88,7 +88,7 @@ PKG_PROG_PKG_CONFIG
 # 4. If any interfaces have been added, removed, or changed since the last update, increment current, and set revision to 0.
 # 5. If any interfaces have been added since the last public release, then increment age.
 # 6. If any existing interfaces have been removed or changed since the last public release, then set age to 0. 
-AC_SUBST([LIBPSL_SO_VERSION], [5:2:0])
+AC_SUBST([LIBPSL_SO_VERSION], [6:0:0])
 AC_SUBST([LIBPSL_VERSION], $VERSION)
 
 # Check for enable/disable builtin PSL data
@@ -243,6 +243,11 @@ else
   TESTS_INFO="Valgrind testing not enabled"
 fi
 
+# Check for distribution-wide PSL file
+AC_ARG_WITH(psl-distfile,
+  AC_HELP_STRING([--with-psl-distfile=[PATH]], [path to distribution-wide PSL file]),
+  PSL_DISTFILE=$withval AC_SUBST(PSL_DISTFILE))
+
 # Check for custom PSL file
 AC_ARG_WITH(psl-file,
   AC_HELP_STRING([--with-psl-file=[PATH]], [path to PSL file]),
@@ -283,6 +288,7 @@ AC_MSG_NOTICE([Summary of build options:
   Libs:              ${LIBS}
   Runtime:           ${enable_runtime}
   Builtin:           ${enable_builtin}
+  PSL Dist File:     ${PSL_DISTFILE}
   PSL File:          ${PSL_FILE}
   PSL Test File:     ${PSL_TESTFILE}
   Tests:             ${TESTS_INFO}

docs/libpsl/libpsl-sections.txt

@@ -13,6 +13,7 @@ psl_error_t
 psl_ctx_t
 psl_load_file
 psl_load_fp
+psl_latest
 psl_builtin
 psl_free
 psl_is_public_suffix
@@ -27,6 +28,7 @@ psl_builtin_sha1sum
 psl_builtin_filename
 psl_builtin_outdated
 psl_is_cookie_domain_acceptable
+psl_dist_filename
 psl_get_version
 psl_check_version_number
 psl_str_to_utf8lower

include/libpsl.h.in

@@ -91,6 +91,10 @@ psl_ctx_t *
 const psl_ctx_t *
 	psl_builtin(void);
 
+/* retrieves most recent PSL data */
+psl_ctx_t *
+	psl_latest(const char *fname);
+
 /* checks whether domain is a public suffix or not */
 int
 	psl_is_public_suffix(const psl_ctx_t *psl, const char *domain);
@@ -139,6 +143,10 @@ const char *
 const char *
 	psl_builtin_filename(void);
 
+/* returns name of distribution PSL data file */
+const char *
+	psl_dist_filename(void);
+
 /* returns library version string */
 const char *
 	psl_get_version(void);

libpsl.pc.in

@@ -4,7 +4,7 @@ libdir=@libdir@
 includedir=@includedir@
 
 Name: @PACKAGE_NAME@
-Description: Publix Suffix List C library.
+Description: Public Suffix List C library.
 Version: @PACKAGE_VERSION@
 URL: @PACKAGE_URL@
 Libs: -L${libdir} -lpsl

src/Makefile.am

@@ -7,7 +7,7 @@ CLEANFILES = suffixes_dafsa.c
 lib_LTLIBRARIES = libpsl.la
 
 libpsl_la_SOURCES = psl.c lookup_string_in_fixed_set.c
-libpsl_la_CPPFLAGS = -I$(top_srcdir)/include
+libpsl_la_CPPFLAGS = -I$(top_srcdir)/include -DPSL_DISTFILE=\"$(PSL_DISTFILE)\"
 # include ABI version information
 libpsl_la_LDFLAGS = -version-info $(LIBPSL_SO_VERSION)
 if WITH_LIBICU
@@ -20,23 +20,10 @@ if WITH_LIBIDN
   libpsl_la_LDFLAGS += -lidn -lunistring
 endif
 
-noinst_PROGRAMS = psl2c
-psl2c_SOURCES = psl2c.c lookup_string_in_fixed_set.c
-psl2c_CPPFLAGS = -I$(top_srcdir)/include -DMAKE_DAFSA=\"$(top_srcdir)/src/psl-make-dafsa\"
-if BUILTIN_GENERATOR_LIBICU
-  psl2c_LDADD = $(LIBICU_LIBS)
-endif
-if BUILTIN_GENERATOR_LIBIDN2
-  psl2c_LDADD = @LTLIBICONV@ -lidn2 -lunistring
-endif
-if BUILTIN_GENERATOR_LIBIDN
-  psl2c_LDADD = @LTLIBICONV@ -lidn -lunistring
-endif
-
 # Build rule for suffix_dafsa.c
 # PSL_FILE can be set by ./configure --with-psl-file=[PATH]
-suffixes_dafsa.c: $(PSL_FILE) psl2c$(EXEEXT)
+suffixes_dafsa.c: $(PSL_FILE) $(srcdir)/psl-make-dafsa
-	./psl2c$(EXEEXT) "$(PSL_FILE)" suffixes_dafsa.c
+	$(srcdir)/psl-make-dafsa --output-format=cxx+ "$(PSL_FILE)" suffixes_dafsa.c
 
 EXTRA_DIST = psl-make-dafsa LICENSE.chromium
 

src/lookup_string_in_fixed_set.c

@@ -1,4 +1,4 @@
-/* Copyright 2015 The Chromium Authors. All rights reserved.
+/* Copyright 2015-2016 The Chromium Authors. All rights reserved.
  * Use of this source code is governed by a BSD-style license that can be
  * found in the LICENSE.chromium file.
  *

src/psl-make-dafsa

@@ -221,6 +221,9 @@ The bytes in the generated array has the following meaning:
 """
 
 import sys
+import os.path
+import time
+import hashlib
 
 class InputError(Exception):
   """Exception raised for errors in the input file."""
@@ -498,6 +501,26 @@ def to_cxx(data, codecs):
   text += b'};\n'
   return text
 
+def sha1_file(name):
+  sha1 = hashlib.sha1()
+  with open(name, 'rb') as f:
+    while True:
+        data = f.read(65536)
+        if not data:
+            break
+        sha1.update(data)
+  return sha1.hexdigest()
+
+def to_cxx_plus(data, codecs):
+  """Generates C++ code from a word list plus some variable assignments as needed by libpsl"""
+  text = to_cxx(data, codecs)
+  text += b'static time_t _psl_file_time = %d;\n' % os.stat(psl_input_file).st_mtime
+  text += b'static int _psl_nsuffixes = %d;\n' % psl_nsuffixes
+  text += b'static int _psl_nexceptions = %d;\n' % psl_nexceptions
+  text += b'static int _psl_nwildcards = %d;\n' % psl_nwildcards
+  text += b'static const char _psl_sha1_checksum[] = "%s";\n' % bytes(sha1_file(psl_input_file), **codecs)
+  text += b'static const char _psl_filename[] = "%s";\n' % bytes(psl_input_file, **codecs)
+  return text
 
 def words_to_whatever(words, converter, utf_mode, codecs):
   """Generates C++ code from a word list"""
@@ -511,31 +534,15 @@ def words_to_cxx(words, utf_mode, codecs):
   """Generates C++ code from a word list"""
   return words_to_whatever(words, to_cxx, utf_mode, codecs)
 
+def words_to_cxx_plus(words, utf_mode, codecs):
+  """Generates C++ code from a word list plus some variable assignments as needed by libpsl"""
+  return words_to_whatever(words, to_cxx_plus, utf_mode, codecs)
 
 def words_to_binary(words, utf_mode, codecs):
   """Generates C++ code from a word list"""
   return b'.DAFSA@PSL_0   \n' + words_to_whatever(words, lambda x, _: bytearray(x), utf_mode, codecs)
 
 
-def parse_psl2c(infile, utf_mode, codecs):
-  """Parses file generated by psl2c and extract strings and return code"""
-  lines = [bytes(line.strip(), **codecs) for line in infile]
-
-  for line in lines:
-    if line[-3:-1] != b', ':
-      raise InputError('Expected "domainname, <digit>", found "%s"' % line)
-    # Technically the DAFSA format could support return values in range [0x00-0x1E],
-    # but the values below are the only with a defined meaning.
-    if line[-1] not in b'0123456789ABCDEF':
-      raise InputError('Expected value to be in range [0-9] or [A-F], found "%s"' % line[-1:])
-
-#  with open("gperf.out", 'w') as outfile:
-#    for line in sorted(lines):
-#      outfile.write(line[:-3] + line[-1] + "\n")
-
-  return [line[:-3] + line[-1:] for line in sorted(lines)]
-
-
 def parse_psl(infile, utf_mode, codecs):
   """Parses PSL file and extract strings and return code"""
   PSL_FLAG_EXCEPTION = (1<<0)
@@ -544,6 +551,8 @@ def parse_psl(infile, utf_mode, codecs):
   PSL_FLAG_PRIVATE = (1<<3) # entry of PRIVATE section
   PSL_FLAG_PLAIN = (1<<4) #just used for PSL syntax checking
 
+  global psl_nsuffixes, psl_nexceptions, psl_nwildcards
+
   psl = {}
   section = 0
 
@@ -565,17 +574,21 @@ def parse_psl(infile, utf_mode, codecs):
       continue # skip comments
 
     if line[:1] == b'!':
+      psl_nexceptions += 1
       flags = PSL_FLAG_EXCEPTION | section
       line = line[1:]
     elif line[:1] == b'*':
       if line[1:2] != b'.':
         print('Unsupported kind of rule (ignored): %s' % line)
         continue
+      psl_nwildcards += 1
+      psl_nsuffixes += 1
       flags = PSL_FLAG_WILDCARD | PSL_FLAG_PLAIN | section
       line = line[2:]
     else:
       if not b'.' in line:
         continue # we do not need an explicit plain TLD rule, already covered by implicit '*' rule
+      psl_nsuffixes += 1
       flags = PSL_FLAG_PLAIN | section
 
     punycode = line.decode('utf-8').encode('idna')
@@ -607,9 +620,8 @@ def parse_psl(infile, utf_mode, codecs):
 def usage():
   """Prints the usage"""
   print('usage: %s [options] infile outfile' % sys.argv[0])
-  print('  --input-format=psl2c    infile has been generated by libpsl/psl2c utility (default)')
-  print('  --input-format=psl      infile is a Public Suffix List file')
   print('  --output-format=cxx     Write DAFSA as C/C++ code (default)')
+  print('  --output-format=cxx+    Write DAFSA as C/C++ code plus statistical assignments')
   print('  --output-format=binary  Write DAFSA binary data')
   print('  --encoding=ascii        7-bit ASCII mode')
   print('  --encoding=utf-8        UTF-8 mode (default)')
@@ -622,7 +634,7 @@ def main():
     usage()
 
   converter = words_to_cxx
-  parser = parse_psl2c
+  parser = parse_psl
   utf_mode = True
 
   codecs = dict()
@@ -630,12 +642,11 @@ def main():
     codecs['encoding'] = 'utf-8'
 
   for arg in sys.argv[1:-2]:
+    # Check --input-format for backward compatibility
     if arg.startswith('--input-format='):
       value = arg[15:].lower()
       if value == 'psl':
         parser = parse_psl
-      elif value == 'psl2c':
-        parser = parse_psl2c
       else:
         print("Unknown input format '%s'" % value)
         return 1
@@ -645,6 +656,8 @@ def main():
         converter = words_to_binary
       elif value == 'cxx':
         converter = words_to_cxx
+      elif value == 'cxx+':
+        converter = words_to_cxx_plus
       else:
         print("Unknown output format '%s'" % value)
         return 1
@@ -664,6 +677,14 @@ def main():
     with open(sys.argv[-1], 'wb') as outfile:
       outfile.write(converter(parser(sys.stdin, utf_mode, codecs), utf_mode, codecs))
   else:
+    """Some statistical data for --output-format=cxx+"""
+    global psl_input_file, psl_nsuffixes, psl_nexceptions, psl_nwildcards
+
+    psl_input_file = sys.argv[-2]
+    psl_nsuffixes = 0
+    psl_nexceptions = 0
+    psl_nwildcards = 0
+
     with open(sys.argv[-2], 'r') as infile, open(sys.argv[-1], 'wb') as outfile:
       outfile.write(converter(parser(infile, utf_mode, codecs), utf_mode, codecs))
 

src/psl-make-dafsa.1

@@ -19,14 +19,11 @@ The format of the data read and written by \fBpsl-make-dafsa\fR
 depends on options passed to it.
 .br
 .TP
-\fB\-\-input\-format=\fR[\fIpsl2c\fR|\fIpsl\fR]
+\fB\-\-output\-format=\fR[\fIcxx\fR|\fIcxx+\fR|\fIbinary\fR]
-\fBpsl2c\fR: (default) input is C code generated by libpsl/psl2c
-.br
-\fBpsl\fR: input is standard textual Public Suffix List file
-.TP
-\fB\-\-output\-format=\fR[\fIcxx\fR|\fIbinary\fR]
 \fBcxx\fR: (default) output is C/C++ code
 .br
+\fBcxx+\fR: output is C/C++ code plus statistical assignments (used by libpsl build process)
+.br
 \fBbinary\fR: output is an architecture-independent binary format
 .TP
 \fB\-\-encoding=\fR[\fIutf-8\fR|\fIascii\fR]

src/psl.c

@@ -44,20 +44,6 @@
 #  define _UNUSED
 #endif
 
-/* if this file is included by psl2c, redefine to use requested library for builtin data */
-#ifdef _LIBPSL_INCLUDED_BY_PSL2C
-#	undef WITH_LIBICU
-#	undef WITH_LIBIDN2
-#	undef WITH_LIBIDN
-#	ifdef BUILTIN_GENERATOR_LIBICU
-#		define WITH_LIBICU
-#	elif defined(BUILTIN_GENERATOR_LIBIDN2)
-#		define WITH_LIBIDN2
-#	elif defined(BUILTIN_GENERATOR_LIBIDN)
-#		define WITH_LIBIDN
-#	endif
-#endif
-
 #if ENABLE_NLS != 0
 #	include <libintl.h>
 #	define _(STRING) gettext(STRING)
@@ -68,6 +54,8 @@
 
 #include <sys/types.h>
 #include <sys/stat.h>
+#include <sys/socket.h>
+#include <netinet/in.h>
 #include <unistd.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -181,24 +169,19 @@ struct _psl_ctx_st {
 		utf8 : 1; /* 1: data contains UTF-8 + punycode encoded rules */
 };
 
-/* include the PSL data compiled by 'psl2c' */
+/* include the PSL data generated by psl-make-dafsa */
-#ifndef _LIBPSL_INCLUDED_BY_PSL2C
 #include "suffixes_dafsa.c"
-#else
-	/* if this source file is included by psl2c.c, provide empty builtin data */
-	static const unsigned char kDafsa[1];
-	static time_t _psl_file_time;
-	static int _psl_nsuffixes;
-	static int _psl_nexceptions;
-	static int _psl_nwildcards;
-	static const char _psl_sha1_checksum[] = "";
-	static const char _psl_filename[] = "";
-#endif
 
 /* references to these PSLs will result in lookups to built-in data */
 static const psl_ctx_t
 	_builtin_psl;
 
+#ifdef PSL_DISTFILE
+static const char _psl_dist_filename[] = PSL_DISTFILE;
+#else
+static const char _psl_dist_filename[] = "";
+#endif
+
 static _psl_vector_t *_vector_alloc(int max, int (*cmp)(const _psl_entry_t **, const _psl_entry_t **))
 {
 	_psl_vector_t *v;
@@ -942,7 +925,7 @@ suffix_yes:
  *
  * For cookie domain checking see psl_is_cookie_domain_acceptable().
  *
- * International @domain names have to be either in UTF-8 (lowercase + NFCK) or in ASCII/ACE format (punycode).
+ * International @domain names have to be either in UTF-8 (lowercase + NFKC) or in ASCII/ACE format (punycode).
  * Other encodings likely result in incorrect return values.
  * Use helper function psl_str_to_utf8lower() for normalization @domain.
  *
@@ -973,7 +956,7 @@ int psl_is_public_suffix(const psl_ctx_t *psl, const char *domain)
  * @type specifies the PSL section where to perform the lookup. Valid values are
  * %PSL_TYPE_PRIVATE, %PSL_TYPE_ICANN and %PSL_TYPE_ANY.
  *
- * International @domain names have to be either in UTF-8 (lowercase + NFCK) or in ASCII/ACE format (punycode).
+ * International @domain names have to be either in UTF-8 (lowercase + NFKC) or in ASCII/ACE format (punycode).
  * Other encodings likely result in incorrect return values.
  * Use helper function psl_str_to_utf8lower() for normalization @domain.
  *
@@ -1000,7 +983,7 @@ int psl_is_public_suffix2(const psl_ctx_t *psl, const char *domain, int type)
  * This function finds the longest public suffix part of @domain by the means
  * of the [Mozilla Public Suffix List](https://publicsuffix.org).
  *
- * International @domain names have to be either in UTF-8 (lowercase + NFCK) or in ASCII/ACE format (punycode).
+ * International @domain names have to be either in UTF-8 (lowercase + NFKC) or in ASCII/ACE format (punycode).
  * Other encodings likely result in incorrect return values.
  * Use helper function psl_str_to_utf8lower() for normalization @domain.
  *
@@ -1040,7 +1023,7 @@ const char *psl_unregistrable_domain(const psl_ctx_t *psl, const char *domain)
  * This function finds the shortest private suffix part of @domain by the means
  * of the [Mozilla Public Suffix List](https://publicsuffix.org).
  *
- * International @domain names have to be either in UTF-8 (lowercase + NFCK) or in ASCII/ACE format (punycode).
+ * International @domain names have to be either in UTF-8 (lowercase + NFKC) or in ASCII/ACE format (punycode).
  * Other encodings likely result in incorrect return values.
  * Use helper function psl_str_to_utf8lower() for normalization @domain.
  *
@@ -1082,7 +1065,7 @@ const char *psl_registrable_domain(const psl_ctx_t *psl, const char *domain)
  * This function loads the public suffixes file named @fname.
  * To free the allocated resources, call psl_free().
  *
- * The suffixes are expected to be UTF-8 encoded (lowercase + NFCK) if they are international.
+ * The suffixes are expected to be UTF-8 encoded (lowercase + NFKC) if they are international.
  *
  * Returns: Pointer to a PSL context or %NULL on failure.
  *
@@ -1111,7 +1094,7 @@ psl_ctx_t *psl_load_file(const char *fname)
  * This function loads the public suffixes from a FILE pointer.
  * To free the allocated resources, call psl_free().
  *
- * The suffixes are expected to be UTF-8 encoded (lowercase + NFCK) if they are international.
+ * The suffixes are expected to be UTF-8 encoded (lowercase + NFKC) if they are international.
  *
  * Returns: Pointer to a PSL context or %NULL on failure.
  *
@@ -1290,7 +1273,7 @@ void psl_free(psl_ctx_t *psl)
  * The builtin data also contains punycode entries, one for each international domain name.
  *
  * If the generation of built-in data has been disabled during compilation, %NULL will be returned.
- * When using the builtin psl context, you can provide UTF-8 (lowercase + NFCK) or ASCII/ACE (punycode)
+ * When using the builtin psl context, you can provide UTF-8 (lowercase + NFKC) or ASCII/ACE (punycode)
  * representations of domains to functions like psl_is_public_suffix().
  *
  * Returns: Pointer to the built in PSL data or NULL if this data is not available.
@@ -1370,7 +1353,7 @@ int psl_suffix_wildcard_count(const psl_ctx_t *psl)
 /**
  * psl_builtin_file_time:
  *
- * This function returns the mtime of the Publix Suffix List file that has been built in.
+ * This function returns the mtime of the Public Suffix List file that has been built in.
  *
  * If the generation of built-in data has been disabled during compilation, 0 will be returned.
  *
@@ -1386,7 +1369,7 @@ time_t psl_builtin_file_time(void)
 /**
  * psl_builtin_sha1sum:
  *
- * This function returns the SHA1 checksum of the Publix Suffix List file that has been built in.
+ * This function returns the SHA1 checksum of the Public Suffix List file that has been built in.
  * The returned string is in lowercase hex encoding, e.g. "2af1e9e3044eda0678bb05949d7cca2f769901d8".
  *
  * If the generation of built-in data has been disabled during compilation, an empty string will be returned.
@@ -1403,7 +1386,7 @@ const char *psl_builtin_sha1sum(void)
 /**
  * psl_builtin_filename:
  *
- * This function returns the file name of the Publix Suffix List file that has been built in.
+ * This function returns the file name of the Public Suffix List file that has been built in.
  *
  * If the generation of built-in data has been disabled during compilation, an empty string will be returned.
  *
@@ -1439,6 +1422,23 @@ int psl_builtin_outdated(void)
 	return 0;
 }
 
+/**
+ * psl_dist_filename:
+ *
+ * This function returns the file name of the distribution/system PSL data file.
+ * This file will be considered by psl_latest().
+ *
+ * Return the filename that is set by ./configure --with-psl-distfile, or an empty string.
+ *
+ * Returns: String containing a PSL file name or an empty string.
+ *
+ * Since: 0.16
+ */
+const char *psl_dist_filename(void)
+{
+	return _psl_dist_filename;
+}
+
 /**
  * psl_get_version:
  *
@@ -1510,7 +1510,7 @@ static int _isip(const char *hostname)
  * This helper function checks whether @cookie_domain is an acceptable cookie domain value for the request
  * @hostname.
  *
- * For international domain names both, @hostname and @cookie_domain, have to be either in UTF-8 (lowercase + NFCK)
+ * For international domain names both, @hostname and @cookie_domain, have to be either in UTF-8 (lowercase + NFKC)
  * or in ASCII/ACE (punycode) format. Other encodings or mixing UTF-8 and punycode likely result in incorrect return values.
  *
  * Use helper function psl_str_to_utf8lower() for normalization of @hostname and @cookie_domain.
@@ -1570,8 +1570,8 @@ int psl_is_cookie_domain_acceptable(const psl_ctx_t *psl, const char *hostname,
  * @locale: locale of @str for to lowercase conversion, e.g. 'de' or %NULL
  * @lower: return value containing the converted string
  *
- * This helper function converts a string to UTF-8 lowercase + NFCK representation.
+ * This helper function converts a string to UTF-8 lowercase + NFKC representation.
- * Lowercase + NFCK UTF-8 is needed as input to the domain checking functions.
+ * Lowercase + NFKC UTF-8 is needed as input to the domain checking functions.
  *
  * @lower is set to %NULL on error.
  *
@@ -1592,20 +1592,19 @@ psl_error_t psl_str_to_utf8lower(const char *str, const char *encoding _UNUSED,
 {
 	int ret = PSL_ERR_INVALID_ARG;
 
-	if (lower)
-		*lower = NULL;
-
 	if (!str)
 		return PSL_ERR_INVALID_ARG;
 
 	/* shortcut to avoid costly conversion */
 	if (_str_is_ascii(str)) {
 		if (lower) {
-			char *p;
+			char *p, *tmp;
 
-			if (!(*lower = strdup(str)))
+			if (!(tmp = strdup(str)))
 				return PSL_ERR_NO_MEM;
 
+			*lower = tmp;
+
 			/* convert ASCII string to lowercase */
 			for (p = *lower; *p; p++)
 				if (isupper(*p))
@@ -1652,7 +1651,11 @@ psl_error_t psl_str_to_utf8lower(const char *str, const char *encoding _UNUSED,
 					ret = PSL_SUCCESS;
 					if (lower) {
 						if (str_length < 256) {
-							if (!(*lower = strdup(utf8_lower)))
+							char *tmp = strdup(utf8_lower);
+
+							if (tmp)
+								*lower = tmp;
+							else
 								ret = PSL_ERR_NO_MEM;
 						} else {
 							*lower = utf8_lower;
@@ -1761,3 +1764,70 @@ out:
 
 	return ret;
 }
+
+/* if file is newer than the builtin data, insert it reverse sorted by mtime */
+static int _insert_file(const char *fname, const char **psl_fname, time_t *psl_mtime, int n)
+{
+	struct stat st;
+	int it;
+
+	if (fname && *fname && stat(fname, &st) == 0 && st.st_mtime > _psl_file_time) {
+		/* add file name and mtime to end of array */
+		psl_fname[n] = fname;
+		psl_mtime[n++] = st.st_mtime;
+
+		/* move the new entry to it's correct position */
+		for (it = n - 2; it >= 0 && st.st_mtime > psl_mtime[it]; it--) {
+			psl_fname[it + 1] = psl_fname[it];
+			psl_mtime[it + 1] = psl_mtime[it];
+			psl_fname[it] = fname;
+			psl_mtime[it] = st.st_mtime;
+		}
+	}
+
+	return n;
+}
+
+/**
+ * psl_latest:
+ * @fname: Name of PSL file or %NULL
+ *
+ * This function loads the the latest available PSL data from either
+ * - @fname (application specific filename, may be %NULL)
+ * - location specified during built-time (filename from ./configure --with-psl-distfile)
+ * - built-in PSL data (generated from ./configure --with-psl-file)
+ * - location of built-in data (filename from ./configure --with-psl-file)
+ * 
+ * If none of the above is available, the function returns %NULL.
+ *
+ * To free the allocated resources, call psl_free().
+ *
+ * Returns: Pointer to a PSL context or %NULL on failure.
+ *
+ * Since: 0.16
+ */
+psl_ctx_t *psl_latest(const char *fname)
+{
+	psl_ctx_t *psl;
+	const char *psl_fname[3];
+	time_t psl_mtime[3];
+	int it, ntimes;
+
+	psl_fname[0] = NULL; /* silence gcc 6.2 false warning */
+
+	/* create array of PSL files reverse sorted by mtime (latest first) */
+	ntimes = _insert_file(fname, psl_fname, psl_mtime, 0);
+	ntimes = _insert_file(_psl_filename, psl_fname, psl_mtime, ntimes);
+	ntimes = _insert_file(_psl_dist_filename, psl_fname, psl_mtime, ntimes);
+
+	/* load PSL data from the latest file, falling back to the second recent, ... */
+	for (psl = NULL, it = 0; it < ntimes; it++) {
+		if (psl_mtime[it] > _psl_file_time)
+			if ((psl = psl_load_file(psl_fname[it])))
+				break;
+	}
+
+	/* if file loading failed or there is no file newer than the builtin data,
+	 * then return the builtin data. */
+	return psl ? psl : (psl_ctx_t *) psl_builtin();
+}

src/psl2c.c

@@ -1,320 +0,0 @@
-/*
- * Copyright(c) 2014-2016 Tim Ruehsen
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
- * DEALINGS IN THE SOFTWARE.
- *
- * This file is part of libpsl.
- *
- * Precompile Public Suffix List into a C source file
- *
- * Changelog
- * 22.03.2014  Tim Ruehsen  created
- *
- */
-
-#if HAVE_CONFIG_H
-# include <config.h>
-#endif
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <time.h>
-#include <sys/stat.h>
-#ifdef HAVE_ALLOCA_H
-#	include <alloca.h>
-#endif
-
-#if defined(BUILTIN_GENERATOR_LIBICU) || defined(BUILTIN_GENERATOR_LIBIDN2) || defined(BUILTIN_GENERATOR_LIBIDN)
-#	define _GENERATE_BUILTIN_DATA
-#endif
-
-#include <libpsl.h>
-
-/* here we include the library source code to have access to internal functions and data structures */
-#define _LIBPSL_INCLUDED_BY_PSL2C
-#	include "psl.c"
-#undef _LIBPSL_INCLUDED_BY_PSL2C
-
-#ifdef _GENERATE_BUILTIN_DATA
-
-#if 0
-static int _check_psl(const psl_ctx_t *psl)
-{
-	int it, pos, err = 0;
-
-	/* check if plain suffix also appears in exceptions */
-	for (it = 0; it < psl->suffixes->cur; it++) {
-		_psl_entry_t *e = _vector_get(psl->suffixes, it);
-
-		if (!e->wildcard && _vector_find(psl->suffix_exceptions, e) >= 0) {
-			fprintf(stderr, "Found entry '%s' also in exceptions\n", e->label);
-			err = 1;
-		}
-	}
-
-	/* check if exception also appears in suffix list as plain entry */
-	for (it = 0; it < psl->suffix_exceptions->cur; it++) {
-		_psl_entry_t *e2, *e = _vector_get(psl->suffix_exceptions, it);
-
-		if ((e2 = _vector_get(psl->suffixes, pos = _vector_find(psl->suffixes, e)))) {
-			if (!e2->wildcard) {
-				fprintf(stderr, "Found exception '!%s' also as suffix\n", e->label);
-				err = 1;
-			}
-			/* Two same domains in a row are allowed: wildcard and non-wildcard.
-			 * Binary search find either of them, so also check previous and next entry. */
-			else if (pos > 0 && _suffix_compare(e, e2 = _vector_get(psl->suffixes, pos - 1)) == 0 && !e2->wildcard) {
-				fprintf(stderr, "Found exception '!%s' also as suffix\n", e->label);
-				err = 1;
-			}
-			else if (pos < psl->suffixes->cur - 1 && _suffix_compare(e, e2 = _vector_get(psl->suffixes, pos + 1)) == 0 && !e2->wildcard) {
-				fprintf(stderr, "Found exception '!%s' also as suffix\n", e->label);
-				err = 1;
-			}
-		}
-	}
-
-	/* check if non-wildcard entry is already covered by wildcard entry */
-	for (it = 0; it < psl->suffixes->cur; it++) {
-		const char *p;
-		_psl_entry_t *e = _vector_get(psl->suffixes, it);
-
-		if (e->nlabels > 1 && !e->wildcard && (p = strchr(e->label, '.'))) {
-			_psl_entry_t *e2, *e3, suffix;
-
-			suffix.label = p + 1;
-			suffix.length = strlen(p + 1);
-			suffix.nlabels = e->nlabels - 1;
-
-			e2 = _vector_get(psl->suffixes, pos = _vector_find(psl->suffixes, &suffix));
-
-			if (e2) {
-				if (e2->wildcard) {
-					fprintf(stderr, "Found superfluous '%s' already covered by '*.%s'\n", e->label, e2->label);
-					err = 1;
-				}
-				/* Two same domains in a row are allowed: wildcard and non-wildcard.
-				* Binary search find either of them, so also check previous and next entry. */
-				else if (pos > 0 && _suffix_compare(e2, e3 = _vector_get(psl->suffixes, pos - 1)) == 0 && e3->wildcard) {
-					fprintf(stderr, "Found superfluous '%s' already covered by '*.%s'\n", e->label, e2->label);
-					err = 1;
-				}
-				else if (pos < psl->suffixes->cur - 1 && _suffix_compare(e2, e3 = _vector_get(psl->suffixes, pos + 1)) == 0 && e3->wildcard) {
-					fprintf(stderr, "Found superfluous '%s' already covered by '*.%s'\n", e->label, e2->label);
-					err = 1;
-				}
-			}
-		}
-	}
-
-	return err;
-}
-#endif
-
-static void _print_psl_entries_dafsa(FILE *fpout, const _psl_vector_t *v)
-{
-	FILE *fp;
-	int it;
-
-#ifdef BUILTIN_GENERATOR_LIBICU
-	do {
-		UVersionInfo version_info;
-		char version[U_MAX_VERSION_STRING_LENGTH];
-
-		u_getVersion(version_info);
-		u_versionToString(version_info, version);
-		fprintf(fpout, "/* automatically generated by psl2c (punycode generated with libicu/%s) */\n", version);
-	} while (0);
-#elif defined(BUILTIN_GENERATOR_LIBIDN2)
-	fprintf(fpout, "/* automatically generated by psl2c (punycode generated with libidn2/%s) */\n", idn2_check_version(NULL));
-#elif defined(BUILTIN_GENERATOR_LIBIDN)
-	fprintf(fpout, "/* automatically generated by psl2c (punycode generated with libidn/%s) */\n", stringprep_check_version(NULL));
-#else
-	fprintf(fpout, "/* automatically generated by psl2c (punycode generated internally) */\n");
-#endif
-
-	if ((fp = fopen("in.tmp", "w"))) {
-		for (it = 0; it < v->cur; it++) {
-			_psl_entry_t *e = _vector_get(v, it);
-
-			fprintf(fp, "%s, %X\n", e->label_buf, (int) (e->flags & 0x0F));
-		}
-
-		fclose(fp);
-	}
-
-	if ((it = system(MAKE_DAFSA " in.tmp out.tmp")))
-		fprintf(stderr, "Failed to execute " MAKE_DAFSA "\n");
-
-	if ((fp = fopen("out.tmp", "r"))) {
-		char buf[256];
-
-		while (fgets(buf, sizeof(buf), fp))
-			fputs(buf, fpout);
-
-		fclose(fp);
-	}
-
-	unlink("in.tmp");
-	unlink("out.tmp");
-}
-#endif /* _GENERATE_BUILTIN_DATA */
-
-static int _print_psl_entries_dafsa_binary(const char *fname, const _psl_vector_t *v)
-{
-	FILE *fp;
-	int ret = 0, it, rc;
-	char cmd[256];
-
-	if ((fp = fopen("in.tmp", "w"))) {
-		for (it = 0; it < v->cur; it++) {
-			_psl_entry_t *e = _vector_get(v, it);
-
-			fprintf(fp, "%s, %X\n", e->label_buf, (int) (e->flags & 0x0F));
-		}
-
-		fclose(fp);
-	} else {
-		fprintf(stderr, "Failed to write open 'in.tmp'\n");
-		return 3;
-	}
-
-	snprintf(cmd, sizeof(cmd), MAKE_DAFSA " --output-format=binary in.tmp %s", fname);
-	if ((rc = system(cmd))) {
-		fprintf(stderr, "Failed to execute '%s' (%d)\n", cmd, rc);
-		ret = 2;
-	}
-
-	unlink("in.tmp");
-	return ret;
-}
-
-static void usage(void)
-{
-	fprintf(stderr, "Usage: psl2c [--binary] <infile> <outfile>\n");
-	fprintf(stderr, "  <infile>  is the 'public_suffix_list.dat', lowercase UTF-8 encoded\n");
-	fprintf(stderr, "  <outfile> is the the filename to be generated from <infile>\n");
-	fprintf(stderr, "  --binary  Generate binary DAFSA output (default: C code for psl.c)\n");
-	exit(1);
-}
-
-int main(int argc, const char **argv)
-{
-	FILE *fpout;
-	psl_ctx_t *psl;
-	int ret = 0, argpos = 1, binary = 0;
-
-	if (argc < 3)
-		usage();
-
-	if (strcmp(argv[argpos], "--binary") == 0) {
-		argpos++;
-		binary = 1;
-	}
-
-	if (argc - argpos != 2)
-		usage();
-
-	if (binary) {
-		if (!(psl = psl_load_file(argv[argpos])))
-			return 2;
-
-		ret = _print_psl_entries_dafsa_binary(argv[argpos + 1], psl->suffixes);
-
-		psl_free(psl);
-		return ret;
-	}
-
-#ifdef _GENERATE_BUILTIN_DATA
-	if (!(psl = psl_load_file(argv[argpos])))
-		return 2;
-
-	if (!psl->suffixes || !psl->nsuffixes) {
-		fprintf(stderr, "Failed to load PSL. Please check content of '%s'.\n", argv[argpos]);
-		return 5;
-	}
-
-	/* look for ambiguous or double entries */
-/*	if (_check_psl(psl)) {
-		psl_free(psl);
-		return 5;
-	}
-*/
-	if ((fpout = fopen(argv[argpos + 1], "w"))) {
-		FILE *pp;
-		struct stat st;
-		size_t cmdsize = 16 + strlen(argv[argpos]);
-		char *cmd = alloca(cmdsize), checksum[64] = "";
-		char *abs_srcfile;
-
-		_print_psl_entries_dafsa(fpout, psl->suffixes);
-
-		snprintf(cmd, cmdsize, "sha1sum %s", argv[argpos]);
-		if ((pp = popen(cmd, "r"))) {
-			if (fscanf(pp, "%63[0-9a-zA-Z]", checksum) < 1)
-				*checksum = 0;
-			pclose(pp);
-		}
-
-		if (stat(argv[argpos], &st) != 0)
-			st.st_mtime = 0;
-		fprintf(fpout, "static time_t _psl_file_time = %lu;\n", st.st_mtime);
-
-		fprintf(fpout, "static int _psl_nsuffixes = %d;\n", psl->nsuffixes);
-		fprintf(fpout, "static int _psl_nexceptions = %d;\n", psl->nexceptions);
-		fprintf(fpout, "static int _psl_nwildcards = %d;\n", psl->nwildcards);
-		fprintf(fpout, "static const char _psl_sha1_checksum[] = \"%s\";\n", checksum);
-
-		/* We need an absolute path here, else psl_builtin_outdated() won't work reliable */
-		/* Caveat: symbolic links are resolved by realpath() */
-		if ((abs_srcfile = realpath(argv[argpos], NULL))) {
-			fprintf(fpout, "static const char _psl_filename[] = \"%s\";\n", abs_srcfile);
-			free(abs_srcfile);
-		} else
-			fprintf(fpout, "static const char _psl_filename[] = \"%s\";\n", argv[argpos]);
-
-		if (fclose(fpout) != 0)
-			ret = 4;
-	} else {
-		fprintf(stderr, "Failed to write open '%s'\n", argv[argpos + 1]);
-		ret = 3;
-	}
-
-	psl_free(psl);
-#else
-	if ((fpout = fopen(argv[argpos + 1], "w"))) {
-		fprintf(fpout, "static const unsigned char kDafsa[1];\n");
-		fprintf(fpout, "static time_t _psl_file_time;\n");
-		fprintf(fpout, "static int _psl_nsuffixes = 0;\n");
-		fprintf(fpout, "static int _psl_nexceptions = 0;\n");
-		fprintf(fpout, "static int _psl_nwildcards = 0;\n");
-		fprintf(fpout, "static const char _psl_sha1_checksum[] = \"\";\n");
-		fprintf(fpout, "static const char _psl_filename[] = \"\";\n");
-
-		if (fclose(fpout) != 0)
-			ret = 4;
-	} else {
-		fprintf(stderr, "Failed to write open '%s'\n", argv[argpos + 1]);
-		ret = 3;
-	}
-#endif /* GENERATE_BUILTIN_DATA */
-
-	return ret;
-}

tests/Makefile.am

@@ -28,9 +28,9 @@ TESTS = $(PSL_TESTS)
 # check-local target works in parallel to the tests, so the test suite will likely fail
 BUILT_SOURCES = psl.dafsa psl_ascii.dafsa
 psl.dafsa:
-	$(top_srcdir)/src/psl-make-dafsa --input-format=psl --output-format=binary "$(PSL_FILE)" psl.dafsa
+	$(top_srcdir)/src/psl-make-dafsa --output-format=binary "$(PSL_FILE)" psl.dafsa
 psl_ascii.dafsa:
-	$(top_srcdir)/src/psl-make-dafsa --input-format=psl --output-format=binary --encoding=ascii "$(PSL_FILE)" psl_ascii.dafsa
+	$(top_srcdir)/src/psl-make-dafsa --output-format=binary --encoding=ascii "$(PSL_FILE)" psl_ascii.dafsa
 
 clean-local:
 	rm -f psl.dafsa psl_ascii.dafsa

tests/test-is-public-all.c

@@ -94,7 +94,7 @@ static void test_psl_entry(const psl_ctx_t *psl, const char *domain, int type)
 		} else ok++;
 
 		if (!(strchr(domain, '.'))) {
-			/* TLDs are always expected to be Publix Suffixes */
+			/* TLDs are always expected to be Public Suffixes */
 			if (!(result = psl_is_public_suffix2(psl, domain, PSL_TYPE_PRIVATE))) {
 				failed++;
 				printf("psl_is_public_suffix2(%s, PSL_TYPE_PRIVATE)=%d (expected 1)\n", domain, result);

tests/test-registrable-domain.c

@@ -53,7 +53,7 @@ static int
 static void testx(const psl_ctx_t *psl, const char *domain, const char *encoding, const char *lang, const char *expected_result)
 {
 	const char *result;
-	char *lower;
+	char *lower = NULL;
 	int rc;
 
 	/* just to cover special code paths for valgrind checking */
@@ -67,8 +67,6 @@ static void testx(const psl_ctx_t *psl, const char *domain, const char *encoding
 		/* if we do not runtime support, test failure have to be skipped */
 		failed++;
 		printf("psl_str_to_utf8lower(%s)=%d\n", domain ? domain : "NULL", rc);
-
-		free(lower);
 		return;
 	}
 #endif

tools/psl.c

@@ -72,7 +72,7 @@ int main(int argc, const char *const *argv)
 {
 	int mode = 1;
 	const char *const *arg, *psl_file = NULL, *cookie_domain = NULL;
-	psl_ctx_t *psl = (psl_ctx_t *) psl_builtin();
+	psl_ctx_t *psl = (psl_ctx_t *) psl_latest(NULL);
 
 	/* set current locale according to the environment variables */
 	#include <locale.h>
@@ -120,7 +120,7 @@ int main(int argc, const char *const *argv)
 				printf("psl %s (0x%06x)\n", PACKAGE_VERSION, psl_check_version_number(0));
 				printf("libpsl %s\n", psl_get_version());
 				printf("\n");
-				printf("Copyright (C) 2014-2015 Tim Ruehsen\n");
+				printf("Copyright (C) 2014-2016 Tim Ruehsen\n");
 				printf("License: MIT\n");
 				exit(0);
 			}
@@ -164,14 +164,14 @@ int main(int argc, const char *const *argv)
 				else if (mode == 4) {
 					char *cookie_domain_lower;
 
-					if ((rc = psl_str_to_utf8lower(domain, NULL, NULL, &cookie_domain_lower)) != PSL_SUCCESS)
+					if ((rc = psl_str_to_utf8lower(domain, NULL, NULL, &cookie_domain_lower)) == PSL_SUCCESS) {
-						fprintf(stderr, "%s: Failed to convert cookie domain '%s' to lowercase UTF-8 (%d)\n", domain, cookie_domain, rc);
-					else
 						printf("%s: %d\n", domain, psl_is_cookie_domain_acceptable(psl, lower, cookie_domain));
-
 						free(cookie_domain_lower);
+					} else
+						fprintf(stderr, "%s: Failed to convert cookie domain '%s' to lowercase UTF-8 (%d)\n", domain, cookie_domain, rc);
 				}
 
+				if (rc == PSL_SUCCESS)
 					free(lower);
 			}
 
@@ -197,6 +197,8 @@ int main(int argc, const char *const *argv)
 			printf("%s: %d\n", *arg, psl_is_cookie_domain_acceptable(psl, *arg, cookie_domain));
 	}
 	else if (mode == 99) {
+		printf("dist filename: %s\n", psl_dist_filename());
+
 		if (psl && psl != psl_builtin()) {
 			printf("suffixes: %d\n", psl_suffix_count(psl));
 			printf("exceptions: %d\n", psl_suffix_exception_count(psl));