Compare commits

..

No commits in common. "master" and "libpsl-0.8.1" have entirely different histories.

2301 changed files with 2344 additions and 108596 deletions

1
.github/FUNDING.yml vendored
View File

@ -1 +0,0 @@
github: rockdaboot

View File

@ -1,53 +0,0 @@
name: "CodeQL"
on:
push:
branches: [ "master" ]
pull_request:
branches: [ "master" ]
schedule:
- cron: "49 4 * * 0"
jobs:
analyze:
name: Analyze
runs-on: ubuntu-latest
permissions:
actions: read
contents: read
security-events: write
strategy:
fail-fast: false
matrix:
language: [ python, cpp ]
steps:
- name: Checkout
uses: actions/checkout@v3
with:
submodules: recursive
- name: Install Packages (cpp)
if: ${{ matrix.language == 'cpp' }}
run: |
sudo apt-get update
sudo apt-get install --yes lzip autopoint
- name: Configure (cpp)
if: ${{ matrix.language == 'cpp' }}
run: ./autogen.sh && ./configure --disable-gtk-doc
- name: Initialize CodeQL
uses: github/codeql-action/init@v2
with:
languages: ${{ matrix.language }}
queries: +security-and-quality
- name: Autobuild
uses: github/codeql-action/autobuild@v2
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v2
with:
category: "/language:${{ matrix.language }}"

24
.gitignore vendored
View File

@ -1,19 +1,14 @@
*.exe
*.gcda
*.gcno
*.gcov
*.gz
*.la
*.lo
*.log
*.o
*.lo
*.la
*.exe
*~
*/.deps
*/.libs
*.cache
*.plist
*.stamp
ABOUT-NLS
aclocal.m4
ar-lib
autom4te.cache/
@ -43,22 +38,12 @@ docs/libpsl/libpsl.prerequisites
docs/libpsl/libpsl.signals
docs/libpsl/version.xml
docs/libpsl/xml/
fuzz/*.log
fuzz/*.trs
fuzz/*_fuzzer
fuzz/*_fuzzer.new
gtk-doc.m4
gtk-doc.make
include/libpsl.h
install-sh
lcov/
libpsl.info
libpsl.pc
libtool
ltmain.sh
m4/
msvc/config-msvc.mak
msvc/config.h.win32
Makefile
Makefile.in
missing
@ -79,13 +64,10 @@ po/remove-potcdate.sed
po/stamp-po
src/psl2c
src/suffixes.c
src/suffixes_dafsa.h
stamp-h1
test-driver
tests/*.log
tests/*.trs
tests/psl.dafsa
tests/psl_ascii.dafsa
tests/test-is-cookie-domain-acceptable
tests/test-is-public
tests/test-is-public-all

View File

@ -1,7 +0,0 @@
extraction:
cpp:
prepare:
packages:
- lzip
configure:
command: ./autogen.sh && ./configure --disable-gtk-doc

View File

@ -1,59 +1,29 @@
sudo: false
dist: xenial
language: c
compiler:
- gcc
- clang-3.8
branches:
only:
- master
- coveralls-text
env:
- RUNTIME=libicu
- RUNTIME=libidn2
- RUNTIME=libidn
- RUNTIME=no
addons:
apt:
packages:
- automake
- autoconf
- autopoint
- libtool
- gtk-doc-tools
- gettext
- libidn11-dev
- libidn2-0-dev
- libicu-dev
- libunistring-dev
- lcov
- clang-3.8
- llvm-3.8-dev
- llvm-dev
- valgrind
- clang
# Change this to your needs
script:
- set -e
- sed -i 's/dist-lzip//g' configure.ac
- ./autogen.sh
# CFI doesn't work out on Ubuntu 14.04 (Trusty). They don't include LLVMgold.so for clang-3.8.
# clang-3.4 doesn't know CFI yet.
# The only chance with Travis-CI is to switch to docker and use Ubuntu 16.04 or whatever.
# - if [[ $CC == "gcc" ]]; then CFI=""; else CFI="--enable-cfi"; fi
- if [[ $CC == "clang-3.8" ]]; then UBSAN="--enable-ubsan"; fi
- ./configure $CFI --enable-asan $UBSAN && make -j4 && make check -j4
- ./configure $CFI --enable-runtime=$RUNTIME --enable-builtin=libicu --enable-asan $UBSAN && make clean && make -j4 && make check -j4
- ./configure $CFI --enable-runtime=$RUNTIME --enable-builtin=libidn2 --enable-asan $UBSAN && make clean && make -j4 && make check -j4
- ./configure $CFI --enable-runtime=$RUNTIME --enable-builtin=libidn --enable-asan $UBSAN && make clean && make -j4 && make check -j4
- ./configure $CFI --enable-runtime=$RUNTIME --disable-builtin --enable-asan $UBSAN && make clean && make -j4 && make check -j4
# libidn2 0.9.0 on Trusty has bugs, valgrinding the fuzz corpora fails
# so we use libicu here
- if [[ $RUNTIME == "libicu" ]]; then VALGRIND="--enable-valgrind-tests"; fi
- ./configure --enable-runtime=$RUNTIME --enable-builtin=libicu --enable-gtk-doc $VALGRIND && make clean && make -j4 && make check -j4 || cat fuzz/*_fuzzer.log
- ./configure && make -j4 && make check -j4
- ./configure --enable-runtime=libicu --enable-builtin=libicu && make clean && make -j4 && make check -j4
- ./configure --enable-runtime=libicu --enable-builtin=libidn2 && make clean && make -j4 && make check -j4
- ./configure --enable-runtime=libicu --enable-builtin=libidn && make clean && make -j4 && make check -j4
- ./configure --enable-runtime=libicu --disable-builtin && make clean && make -j4 && make check -j4
- ./configure --enable-runtime=libidn2 --enable-builtin=libicu && make clean && make -j4 && make check -j4
- ./configure --enable-runtime=libidn2 --enable-builtin=libidn2 && make clean && make -j4 && make check -j4
- ./configure --enable-runtime=libidn2 --enable-builtin=libidn && make clean && make -j4 && make check -j4
- ./configure --enable-runtime=libidn2 --disable-builtin && make clean && make -j4 && make check -j4
- ./configure --enable-runtime=libidn --enable-builtin=libicu && make clean && make -j4 && make check -j4
- ./configure --enable-runtime=libidn --enable-builtin=libidn2 && make clean && make -j4 && make check -j4
- ./configure --enable-runtime=libidn --enable-builtin=libidn && make clean && make -j4 && make check -j4
- ./configure --enable-runtime=libidn --disable-builtin && make clean && make -j4 && make check -j4
- ./configure --disable-runtime --enable-builtin=libicu && make clean && make -j4 && make check -j4
- ./configure --disable-runtime --enable-builtin=libidn2 && make clean && make -j4 && make check -j4
- ./configure --disable-runtime --enable-builtin=libidn && make clean && make -j4 && make check -j4
- ./configure --disable-runtime --disable-builtin && make clean && make -j4 && make check -j4
- ./configure --enable-gtk-doc && make -j4 && make check -j4
- make distcheck
- if [[ $CC == "gcc" && $RUNTIME == "libicu" ]]; then ./.travis_coveralls.sh; fi
before_install:
- sudo apt-get -qq update
- sudo apt-get -q install autoconf automake autopoint libtool gtk-doc-tools gettext libidn11 libidn11-dev libidn2-0 libidn2-0-dev libicu48 libicu-dev libunistring0 libunistring-dev

View File

@ -1,6 +0,0 @@
#!/bin/bash
make check-coverage-libicu
pip install --user cpp-coveralls
coveralls --include src/

1282
ABOUT-NLS Normal file

File diff suppressed because it is too large Load Diff

View File

@ -15,10 +15,3 @@ Dagobert Michelsen (Fixed Solaris building)
Christopher Meng (Fedora building)
Jakub Čajka
Giuseppe Scrivano
Ryan Sleevi (Discussion, Requested DAFSA format and ICANN/PRIVATE support)
Daurnimator (Code review, discussion, reports)
Olle Liljenzin (Original DAFSA implementation and UTF-8 patch)
Claudio Saveedra (Add support for PSL_TYPE_NO_STAR_RULE)
Chun-wei Fan (Add NMake files)
Xavier Claessens (Add Meson build system)
Ignacio Casal Quinteiro (Improve Meson build)

View File

@ -1,4 +1,4 @@
Copyright (C) 2014-2022 Tim Rühsen
Copyright (C) 2014 Tim Rühsen
Permission is hereby granted, free of charge, to any person obtaining a
copy of this software and associated documentation files (the "Software"),

View File

@ -1,4 +1,4 @@
Copyright (C) 2014-2022 Tim Rühsen
Copyright (C) 2014 Tim Rühsen
Permission is hereby granted, free of charge, to any person obtaining a
copy of this software and associated documentation files (the "Software"),

View File

@ -1,14 +1,6 @@
# got some hints from https://gitorious.org/openismus-playground/examplelib/source
SUBDIRS = po include src tools
if ENABLE_GTK_DOC
SUBDIRS += docs/libpsl
else
if ENABLE_MAN
SUBDIRS += docs/libpsl
endif
endif
SUBDIRS += fuzz tests msvc
SUBDIRS = po include src tools $(LIBPSL_DOCS) tests
ACLOCAL_AMFLAGS = -I m4 ${ACLOCAL_FLAGS}
@ -22,46 +14,4 @@ ACLOCAL_AMFLAGS = -I m4 ${ACLOCAL_FLAGS}
pkgconfigdir = $(libdir)/pkgconfig
pkgconfig_DATA = libpsl.pc
EXTRA_DIST = build-aux/config.rpath LICENSE meson.build meson_options.txt
dist-hook:
mkdir -p $(distdir)/list/tests
cp -p $(PSL_FILE) $(distdir)/list
cp -p $(PSL_TESTFILE) $(distdir)/list/tests
clean-local:
rm -rf */*.gc?? */*/*.gc?? libpsl.info lcov
LCOV_INFO=libpsl.info
check-coverage: clean
if test -z "$(XLIB)"; then \
CFLAGS=$$CFLAGS" --coverage -O0" LDFLAGS=$$LDFLAGS" --coverage" ./configure --disable-runtime --disable-builtin; \
else \
CFLAGS=$$CFLAGS" --coverage -O0" LDFLAGS=$$LDFLAGS" --coverage" ./configure --enable-runtime=$(XLIB) --enable-builtin=$(XLIB); \
fi
$(MAKE)
lcov --capture --initial --directory src --output-file $(LCOV_INFO)
$(MAKE) check
lcov --capture --directory src --output-file $(LCOV_INFO)
genhtml --prefix . $(LCOV_INFO) --legend --title "libpsl" --output-directory=lcov
@echo
@echo "You can now view the coverage report with 'xdg-open lcov/index.html'"
check-coverage-libidn:
XLIB=libidn $(MAKE) check-coverage
check-coverage-libidn2:
XLIB=libidn2 $(MAKE) check-coverage
check-coverage-libicu:
XLIB=libicu $(MAKE) check-coverage
fuzz-coverage: clean
$(MAKE) -C src CFLAGS="$(CFLAGS) --coverage" LDFLAGS="$(LDFLAGS) --coverage"
$(MAKE) -C fuzz fuzz-coverage CFLAGS="$(CFLAGS) --coverage" LDFLAGS="$(LDFLAGS) --coverage"
lcov --capture --initial --directory src --directory fuzz --output-file $(LCOV_INFO)
lcov --capture --directory src --directory fuzz --output-file $(LCOV_INFO)
# lcov --remove $(LCOV_INFO) '*/test_linking.c' '*/css_tokenizer.lex' '*/<stdout>' -o $(LCOV_INFO)
genhtml --prefix . --ignore-errors source $(LCOV_INFO) --legend --title "libpsl-fuzz" --output-directory=lcov
@echo
@echo "You can now view the coverage report with 'xdg-open lcov/index.html'"
EXTRA_DIST = config.rpath LICENSE list/public_suffix_list.dat list/tests/test_psl.txt

View File

@ -1,35 +0,0 @@
USE_CLIB2=YES
ifeq ($(USE_CLIB2), YES)
LIBC=clib2
else
LIBC=newlib
endif
all: build
init:
# git submodule update --init
sh ./autogen.sh
autoreconf --install --force --symlink
build: init
CC="ppc-amigaos-gcc" CFLAGS="-mcrt=${LIBC} -fPIC" CXXFLAGS="-mcrt=${LIBC} -fPIC" LDFlags="-mcrt=${LIBC}" LIBS="-lpthread -athread=native" ./configure --host=ppc-amigaos --enable-shared=no --enable-builtin --disable-runtime && \
make -j$(shell nproc)
clean:
@make -f Makefile clean
@rm -rf autom4te.cache .deps
@rm config.log config.status configure libtool Makefile Makefile.in *.m4
release:
@mkdir -p release/local/common/include
@mkdir -p release/local/clib2/lib
# @mkdir -p release/local/newlib/lib
@make -f Makefile.os4 USE_CLIB2=YES
@cp ./src/.libs/libpsl.a ./src/.libs/libpsl.lai ./src/libpsl.la release/local/clib2/lib/
@cp ./include/libpsl.h release/local/common/include/
# @make -f Makefile.os4 clean
# @make -f Makefile.os4 USE_CLIB2=NO
# @cp ./src/.libs/libpsl.a ./src/.libs/libpsl.lai ./src/libpsl.la release/local/newlib/lib/
@lha -aeq libpsl.lha release/

139
NEWS
View File

@ -1,141 +1,4 @@
Copyright (C) 2014-2022 Tim Rühsen
26.12.2022 Release V0.21.2
* Increased internal label size
* Meson build improvements (needs meson >= 0.60.0)
* Autoconf build improvements
* Add instructions on how to build from tarball
* Add WSAStartup() for Windows psl tool and tests
* Bump gettext version to 0.19.3
* Fix stack buffer overflow WRITE 1 in domain_to_punycode()
(Relevant only when built without any IDNA library.)
* Fix undefined behavior in library code
* Ensures that calls to fopen() and stat() can handle largefiles
* Several minor (non-functional) changes
18.07.2020 Release V0.21.1
* Fixing a test due to recent changes in upstream PSL
* Meson build improvements
* Documentation cleanups
* Use semantic versioning for git tags
16.04.2019 Release V0.21.0
* Add -b/--batch to 'psl' to suppress printing the domain
* Add support for Meson build system
* Improve build system
* Improve Windows compatibility
* Remove NLS / gettext
* Several cleanups and cosmetics
26.04.2018 Release V0.20.2
* Fix non srcdir builds
* Add API decoration
* Fix for MSVC/Win32 builds
* Detection fallback from libidn2 to libcu, libidn
* Fix MinGW cross builds on Linux
* Add NMake Makefiles for Visual Studio builds
26.02.2018 Release V0.20.1
* Fix issue introduced with PSL_TYPE_NO_STAR_RULE in V0.20.0
* Fix SO_VERSION to 8:0:3
* Improve unit tests
22.02.2018 Release V0.20.0
* Remove hard-coded gcc flag in Makefile.am
* Prevent excessive CPU cycles on large inputs
* New flag PSL_TYPE_NO_STAR_RULE to skip star rule
09.11.2017 Release V0.19.1
* Add the forgotten commit with NEWS and configure.ac
09.11.2017 Release V0.19.0
* New function psl_free_string()
* psl_make_dafsa now works with python2 and python3
* psl_*count() functions now return -1 if info is not available
* Fixed unsigned integer overflow in _mem_is_ascii()
* Add -fsanitize-address-use-after-scope to --enable-asan if available
20.07.2017 Release V0.18.0
* Fix order of files in psl_latest()
* Add fuzzing architecture
* Fix memleak in _psl_is_public_suffix()
* Add configure option --enable-asan (Address sanitizer)
* Add configure option --enable-usan (Undefined sanitizer)
* Add configure option --enable-cfi (Control Flow Integrity)
* Fix finding libidn2 for static builds
* Fix use of uninitialized stack value
* Fix buffer overflow in libicu build
* Use libidn2 as default for builds (former libicu)
* Add pkg-config support for libidn and libidn2
16.01.2017 Release V0.17.0
* Use TR46 non-transitional for IDNA (libicu, libidn2 >= 0.14)
* Fix coverage upload from TravisCI to Coveralls
* New tests to cover psl_latest() and psl_dist_filename()
15.12.2016 Release V0.16.1
* Fix SO_VERSION to 6:0:1
* Add --use-latest-psl to tools/psl as default
16.12.2016 Release V0.16.0
* Add functions psl_latest() and psl_dist_filename()
* Do not taint out variable on error in psl_str_to_utf8lower()
* Replace psl2c by psl-make-dafsa
* Add missing includes for OpenBSD
* Fix typos
* Update copyright year
14.11.2016 Release V0.15.0
* Python3 compatibility for psl-make-dafsa
* Support for UTF-8 in DAFSA data
* Skip punycode conversion if DAFSA has UTF-8
* Better code coverage by test suite
* Code cleanup and enhancements
* Install man pages for psl-make-dafsa and psl
* Enhancements to the documentation
30.07.2016 Release V0.14.0
* Remove unneeded libraries from tools/psl link step
* Use https instead of http where possible
* Add man page for tools/psl
* Add header magic to DAFSA files
* Rename make_dafsa.py to psl-make-dafsa
* Add man page for psl-make-dafsa
02.03.2016 Release V0.13.0
* Use tests.txt as PSL test file by default
* Slightly shorter DAFSA array when sorting input
* Check for python 2.7+ in configure.ac
* Fix python3 incompatibilities in make_dafsa.py
02.01.2016 Release V0.12.0
* Load DAFSA binaries via psl_load_file() via auto-detection
* Add more tests
* Remove psl_builtin_compile_time()
* Compile PSL into DAFSA using make_dafsa.py
* Avoid libicu dependency with --enable-runtime=no
* Test on new Travis-CI build farm
* Use DAFSA format for builtin PSL data
* Add function psl_is_public_suffix2()
* Fix psl_builtin_outdated()
* Fix several bugs
* Cleanup code
23.09.2015 Release V0.11.0
* Add new function psl_check_version_number()
* Add version defines to include file
19.09.2015 Release V0.10.0
* Code simplified
* Less data entries, faster lookups
* Add new function psl_suffix_wildcard_count()
* Add new helper function psl_builtin_outdated()
15.09.2015 Release V0.9.0
* Added semantic checks to PSL entries when generating built-in data
* Fix test suite for TLD exceptions (not used yet in reality)
* Removed wrong assumption from test suite
* Support explicit combination of 'foo.bar' and '*.foo.bar'
Copyright (C) 2014 Tim Rühsen
14.08.2015 Release V0.8.1
* Fix documentation

View File

@ -1,138 +0,0 @@
Building libpsl with Visual Studio
==================================
Building libpsl for Windows using Visual Studio 2008 or later is
supported with NMake (from release tarballs) or Meson (from GIT
checkouts). The following sections will cover building libpsl with
these methods.
Currently, for builtin/runtime public suffix list (PSL) IDNA handling,
only ICU is supported for Visual Studio builds.
Using NMake (from a release tarball)
===========
You will need a Python 2.7.x or later installation in order to
complete the build successfully.
You will need the ICU (International Components for Unicode)
libraries, headers and DLLs, to build libpsl, unless both
DISABLE_BUILTIN=1 and DISABLE_RUNTIME=1 are passed into the NMake
command line as listed below.
You can also buid libpsl with libiconv and gettext support, please
see the options below for enabling such support.
In a Visual Studio command prompt which matches your desired
configuration (x86/Win32, x64 etc.),
go to $(srcroot)\msvc, and issue the following command:
nmake /f Makefile.vc CFG=[debug|release]
A 'test' target is provided to build the test programs, while a
'clean' target is provided to remove all the compiled and generated
files for the build. An 'install' target is provided to copy the
build PSL DLL, .lib and executables, as well as the related PDB files
and libpsl header, into appropriate locations under PREFIX (please see
below).
This will build the libpsl DLL/LIB and the psl.exe utility in the
vsX\$(CFG)\$(ARCH) subdirectory, where X is the release version
of Visual Studio, such as 9 for 2008 and 16 for 2019, and ARCH is
Win32 for 32-bit builds and x64 for 64-bit (x86_64) builds.
A number of options can be passed into the NMake command, as follows.
Enable by setting each option to 1, unless otherwise indicated:
* PSL_FILE: Location of the PSL data file, which is retrieved from
https://publicsuffix.org/list/public_suffix_list.dat,
or some other custom location (not supported). Default
is in $(srcroot)\list\public_suffix_list.dat. This is
needed to generate the suffixes_dafsa.h header required
for the build, as well as the binary and ascii dafsa
files used for the test programs.
* TEST_PSL_FILE: Location of the test PSL file. Default is in
$(srcroot)\list\tests\tests.txt. This is
required for building and running the test
programs.
* STATIC: Set if building static versions of libpsl is desired.
* USE_LIBTOOL_DLLNAME: Set to use libtool-style DLL naming.
* DISABLE_RUNTIME: Do not use ICU to generate runtime PSL data.
* DISABLE_BUILTIN: Do not use ICU to generate builtin PSL data.
* USE_ICONV: Enable libiconv support, requires libiconv.
* USE_GETTEXT: Enable gettext support for displaying i18n messages.
Implies USE_ICONV, and requires gettext-runtime.
* PYTHON: Full path to a Python 2.7.x (or later) interpreter, if not
already in your PATH.
Required to generate DAFSA headers and data files that is
needed for the build, as well as generating pkg-config
files for NMake builds.
* PREFIX: Base installation path of the build. Note that any dependent
libraries are searched first from the include\ and lib\
sub-directories in PREFIX before searching in the paths
specified by %INCLUDE% and %LIB%. Default is
$(srcroot)\..\vsX\$(PLATFORM), where X is the release version
of Visual Studio, such as 9 for 2008 and 16 for 2019,
$(PLATFORM) is the target platform (Win32/x64) of the build.
Building libpsl with Meson
==========================
Building using Meson is now supported for Visual Studio builds from a
GIT checkout.
Besides the requirements listed in the NMake builds, you will also need
* Python 3.5.x or later
* Meson build system, use PIP to install from Python 3.5.x64
* Ninja build tool (if not involking Meson with --backend=
vs[2010|2015|2017|2019])
* A compatible PSL data file and a test PSL data file. You may
consider using the ones shipped with the latest libpsl release
tarball and place the PSL data file in $(srcroot)/list and the
test PSL data file in $(srcroot)/list/tests. You may also choose
to download the latest PSL data file from
https://publicsuffix.org/list/public_suffix_list.dat and place it
it $(srcroot)/list. Alternatively, specify
-Dpsl_file=<path_to_psl_data_file> and/or
-Dpsl_testfile=<path_to_test_psl_data_file> when invoking Meson.
Open a Visual Studio command prompt and enter an empty build directory.
Your Python interpreter, Meson executable script and Ninja (if used)
need to be in your PATH.
Any dependent libraries that are being used should have their headers
found in paths specified by %INCLUDE% and their .lib files in the
paths specified by %LIB%.
In the empty build directory, run the following:
meson <path_to_libpsl_git_checkout> --buildtype=... --prefix=<some_prefix> [--backend=vs[2010|2015|2017|2019]]
Please see the Meson documentation for the values accepted by
--buildtype. --backend=vsXXXX generates the corresponding versions
of the Visual Studio solution files to build libpsl, which
will elimnate the need to have the Ninja build tool installed.
When the Meson configuration completes, run 'ninja' or open the
generated solution files with Visual Studio and build the projects
to carry out the build. Run 'ninja test' or the test project to
test the build and run 'ninja install' or 'ninja install' to
install the build results.
If building with Visual Studio 2008, run the following after running
'ninja install' in your builddir:
for /r %f in (*.dll.manifest) do if exist $(prefix)\bin\%~nf mt /manifest %f /outputresource:$(prefix)\bin\%~nf;2
for /r %f in (*.exe.manifest) do if exist $(prefix)\bin\%~nf mt /manifest %f /outputresource:$(prefix)\bin\%~nf;1
So that the application manifests get properly embedded.

View File

@ -1,33 +1,20 @@
[![Travis-CI Status](https://travis-ci.org/rockdaboot/libpsl.png?branch=master)](https://travis-ci.org/rockdaboot/libpsl)
[![Coverity Scan](https://img.shields.io/coverity/scan/10227.svg)](https://scan.coverity.com/projects/rockdaboot-libpsl)
[![Coverage Status](https://coveralls.io/repos/github/rockdaboot/libpsl/badge.svg?branch=master)](https://coveralls.io/github/rockdaboot/libpsl?branch=master)
Solaris OpenCSW [![Build Status Solaris amd64](https://buildfarm.opencsw.org/buildbot/png?builder=libpsl-solaris10-amd64)](https://buildfarm.opencsw.org/buildbot/builders/libpsl-solaris10-amd64)
[![Build Status Solaris i386](https://buildfarm.opencsw.org/buildbot/png?builder=libpsl-solaris10-i386)](https://buildfarm.opencsw.org/buildbot/builders/libpsl-solaris10-i386)
[![Build Status Solaris Sparc](https://buildfarm.opencsw.org/buildbot/png?builder=libpsl-solaris10-sparc)](https://buildfarm.opencsw.org/buildbot/builders/libpsl-solaris10-sparc)
[![Build Status Solaris SparcV9](https://buildfarm.opencsw.org/buildbot/png?builder=libpsl-solaris10-sparcv9)](https://buildfarm.opencsw.org/buildbot/builders/libpsl-solaris10-sparcv9)
[![Build Status](https://travis-ci.org/rockdaboot/libpsl.png?branch=master)](https://travis-ci.org/rockdaboot/libpsl)
libpsl - C library to handle the Public Suffix List
===================================================
A *Public Suffix List* is a collection of Top Level Domains (TLDs) suffixes.
TLDs include *Global Top Level Domains* (gTLDs) like `.com` and `.net`;
*Country Top Level Domains* (ccTLDs) like `.de` and `.cn`;
and *[Brand Top Level Domains](https://icannwiki.org/Brand_TLD)* like `.apple` and `.google`.
Brand TLDs allows users to register their own top level domain that exist at the same level as ICANN's gTLDs.
Brand TLDs are sometimes referred to as Vanity Domains.
A "public suffix" is a domain name under which Internet users can directly register own names.
Browsers, web clients and other user agents can use a public suffix list to:
Browsers and other web clients can use it to
- avoid privacy-leaking "supercookies"
- avoid privacy-leaking "super domain" certificates ([see post from Jeffry Walton](https://lists.gnu.org/archive/html/bug-wget/2014-03/msg00093.html))
- avoid privacy-leaking "super domain" certificates ([see post from Jeffry Walton](http://lists.gnu.org/archive/html/bug-wget/2014-03/msg00093.html))
- domain highlighting parts of the domain in a user interface
- sorting domain lists by site
Libpsl...
- has built-in PSL data for fast access (DAWG/DAFSA reduces size from 180kB to ~32kB)
- has built-in PSL data for fast access
- allows to load PSL data from files
- checks if a given domain is a "public suffix"
- provides immediate cookie domain verification
@ -35,13 +22,11 @@ Libpsl...
- finds the shortest private part of a given domain
- works with international domains (UTF-8 and IDNA2008 Punycode)
- is thread-safe
- handles IDNA2008 UTS#46 (if libicu is available)
- handles IDNA2008 UTS#46 (libicu is used by psl2c if installed)
Find more information about the Public Suffix List [here](https://publicsuffix.org/).
Find more information about the Publix Suffix List [here](http://publicsuffix.org/).
Download the Public Suffix List [here](https://github.com/publicsuffix/list/blob/master/public_suffix_list.dat).
The original DAFSA code is from the [Chromium Project](https://code.google.com/p/chromium/).
Download the Public Suffix List [here](https://hg.mozilla.org/mozilla-central/raw-file/tip/netwerk/dns/effective_tld_names.dat).
API Documentation
@ -83,80 +68,35 @@ library API via command line.
prints the usage.
Convert PSL into DAFSA
----------------------
The [DAFSA](https://en.wikipedia.org/wiki/Deterministic_acyclic_finite_state_automaton) format is a compressed
representation of strings. Here we use it to reduce the whole PSL to about 32k in size.
Generate `psl.dafsa` from `list/public_suffix_list.dat`
$ src/psl-make-dafsa --output-format=binary list/public_suffix_list.dat psl.dafsa
Test the result (example)
$ tools/psl --load-psl-file psl.dafsa aeroclub.aero
License
-------
Libpsl is made available under the terms of the MIT license.<br>
See the LICENSE file that accompanies this distribution for the full text of the license.
src/psl-make-dafsa and src/lookup_string_in_fixed_set.c are licensed under the term written in
src/LICENSE.chromium.
Building from tarball
---------------------
Choose a release from https://github.com/rockdaboot/libpsl/tags an download the tarball
named `libpsl-{version}.tar.*`. Unpack with `tar xf <filename>`, cd into the libsl* directory.
Build with
```
./configure
make
make check
```
Install with
```
sudo make install
```
Building from git
-----------------
You should have python2.7+ installed.
Download project and prepare sources with
git clone https://github.com/rockdaboot/libpsl
git clone http://github.com/rockdaboot/libpsl
./autogen.sh
./configure
make
make check
If you prefer a `meson` build
meson builddir
ninja -C builddir
ninja -C builddir test
There is also an unofficial MSVC nmake build configuration in `msvc/`. Please
see README.MSVC.md on building libpsl with Visual Studio via NMake or Meson.
Mailing List
------------
[Mailing List Archive](https://groups.google.com/forum/#!forum/libpsl-bugs)
[Mailing List Archive](http://news.gmane.org/gmane.network.dns.libpsl.bugs)
[Mailing List](https://groups.google.com/forum/#!forum/libpsl-bugs)
To join the mailing list send an email to
libpsl-bugs+subscribe@googlegroups.com
<libpsl-bugs+subscribe@googlegroups.com>
and follow the instructions provided by the answer mail.

View File

@ -1,21 +1,21 @@
#!/bin/sh
# !/bin/sh -e
AUTORECONF=$(which autoreconf 2>/dev/null)
AUTORECONF=`which autoreconf 2>/dev/null`
if test $? -ne 0; then
echo "No 'autoreconf' found. You must install the autoconf package."
exit 1
fi
GIT=$(which git 2>/dev/null)
GIT=`which git 2>/dev/null`
if test $? -ne 0; then
echo "No 'git' found. You must install the git package."
exit 1
fi
# create m4 before gtkdocize
mkdir -p m4 2>/dev/null
# create m4 before gtkdocize
mkdir m4 2>/dev/null
GTKDOCIZE=$(which gtkdocize 2>/dev/null)
GTKDOCIZE=`which gtkdocize 2>/dev/null`
if test $? -ne 0; then
echo "No gtk-doc support found. You can't build the docs."
# rm because gtk-doc.make might be a link to a protected file
@ -24,12 +24,12 @@ if test $? -ne 0; then
echo "CLEANFILES =" >>gtk-doc.make
GTKDOCIZE=""
else
$GTKDOCIZE
$GTKDOCIZE || exit $?
fi
$GIT submodule init
$GIT submodule update
$AUTORECONF --install --force --symlink
$AUTORECONF --install --force --symlink || exit $?
echo
echo "----------------------------------------------------------------"

View File

@ -1,7 +1,7 @@
AC_INIT([libpsl],[m4_normalize(m4_include(version.txt))],[tim.ruehsen@gmx.de],[libpsl],[https://github.com/rockdaboot/libpsl])
AC_INIT([libpsl], [0.8.1], [tim.ruehsen@gmx.de], [libpsl], [http://github.com/rockdaboot/libpsl])
AC_PREREQ([2.59])
AC_CONFIG_AUX_DIR([build-aux])
AM_INIT_AUTOMAKE([1.10 no-define foreign dist-lzip])
AM_INIT_AUTOMAKE([1.10 -Wall no-define foreign])
# Generate two configuration headers; one for building the library itself with
# an autogenerated template, and a second one that will be installed alongside
@ -10,33 +10,18 @@ AC_CONFIG_HEADERS([config.h])
AC_PROG_CC
m4_ifdef([AM_PROG_AR], [AM_PROG_AR])
#LT_INIT([disable-static])
LT_INIT([win32-dll])
LT_INIT
AC_CONFIG_MACRO_DIR([m4])
m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])])
AC_SYS_LARGEFILE
dnl Check that compiler understands inline
AC_C_INLINE
dnl Check for visibility support
gl_VISIBILITY
#
# Generate version defines for include file
#
AC_SUBST([LIBPSL_VERSION_MAJOR], [`echo $VERSION|cut -d'.' -f1`])
AC_SUBST([LIBPSL_VERSION_MINOR], [`echo $VERSION|cut -d'.' -f2`])
AC_SUBST([LIBPSL_VERSION_PATCH], [`echo $VERSION|cut -d'.' -f3`])
AC_SUBST([LIBPSL_VERSION_NUMBER], [`printf '0x%02x%02x%02x' $LIBPSL_VERSION_MAJOR $LIBPSL_VERSION_MINOR $LIBPSL_VERSION_PATCH`])
AC_CONFIG_FILES([include/libpsl.h])
#
# Gettext
#
AM_GNU_GETTEXT([external],[need-ngettext])
AM_GNU_GETTEXT_VERSION([0.19.3])
AM_GNU_GETTEXT_REQUIRE_VERSION([0.19.3])
AM_GNU_GETTEXT_VERSION([0.18.1])
#
# check for gtk-doc
@ -52,74 +37,38 @@ AM_CONDITIONAL([GTK_DOC_USE_LIBTOOL], false)
])
if test x"$have_gtk_doc" = xyes -a x"$enable_gtk_doc" = xyes; then
AC_CONFIG_FILES([docs/libpsl/Makefile docs/libpsl/version.xml])
AC_SUBST([LIBPSL_DOCS], [docs/libpsl])
fi
#
# enable creation of man pages
#
AC_ARG_ENABLE([man],
[AS_HELP_STRING([--enable-man], [generate man pages [default=auto]])],
[
if test "$enable_man" = yes; then
AC_PATH_PROG([XSLTPROC], [xsltproc])
AS_IF([test -z "$XSLTPROC"], [
AC_MSG_ERROR([xsltproc is required for --enable-man])
enable_man="no (xsltproc not found)"
])
fi
], [ enable_man=no ])
AM_CONDITIONAL(ENABLE_MAN, test x$enable_man = xyes)
# src/psl-make-dafsa needs python 2.7+
AM_PATH_PYTHON([2.7])
AC_ARG_ENABLE(man,[AC_HELP_STRING([--enable-man],
[generate man pages [default=auto]])],enable_man=yes,enable_man=no)
AS_IF([test "$enable_man" != no], [
AC_PATH_PROG([XSLTPROC], [xsltproc])
AS_IF([test -z "$XSLTPROC"], [
AS_IF([test "$enable_man" = yes], [
AC_MSG_ERROR([xsltproc is required for --enable-man])
])
enable_man=no
])
])
AM_CONDITIONAL(ENABLE_MAN, test x$enable_man != xno)
AC_MSG_CHECKING([whether to generate man pages])
AS_IF([ test "$enable_man" != no ], [
AC_MSG_RESULT([yes])
], [
AC_MSG_RESULT([no])
])
PKG_PROG_PKG_CONFIG
AC_ARG_ENABLE([fuzzing],
[AS_HELP_STRING([--enable-fuzzing], [Turn on fuzzing build (for developers)])],
[enable_fuzzing=yes;
AC_SUBST([LIB_FUZZING_ENGINE])
AC_DEFINE([FUZZING], 1, [Define to 1 if this is a fuzzing build])
], [enable_fuzzing=no; LIB_FUZZING_ENGINE=""])
AM_CONDITIONAL([FUZZING], [test "$enable_fuzzing" = "yes"])
AC_ARG_ENABLE([cfi],
[AS_HELP_STRING([--enable-cfi], [Turn on clang's Control Flow Integrity (CFI)])],
[
if test "$enable_cfi" = yes; then
CFLAGS=$CFLAGS" -B/usr/bin/gold -fsanitize=cfi -flto -fvisibility=default -fno-sanitize-trap=all"
AC_LINK_IFELSE([
AC_LANG_PROGRAM([], [])
], [], [
AC_MSG_ERROR([clang 3.7+ and the 'gold' linker are required for --enable-cfi])
])
fi
], [ enable_cfi=no ])
AC_ARG_ENABLE([ubsan],
[AS_HELP_STRING([--enable-ubsan], [Turn on Undefined Behavior Sanitizer (UBSan)])],
[
if test "$enable_ubsan" = yes; then
# Set basic UBSAN compiler flags. Add your own to CFLAGS.
CFLAGS=$CFLAGS" -fsanitize=undefined -fno-sanitize-recover=undefined"
fi
], [ enable_ubsan=no ])
AC_ARG_ENABLE([asan],
[AS_HELP_STRING([--enable-asan], [Turn on Address Sanitizer (ASan)])],
[
if test "$enable_asan" = yes; then
# Set basic ASAN compiler flags. Add your own to CFLAGS.
CFLAGS=$CFLAGS" -fsanitize=address -fno-omit-frame-pointer"
AX_CHECK_COMPILE_FLAG([-fsanitize-address-use-after-scope], [CFLAGS="$CFLAGS -fsanitize-address-use-after-scope"])
fi
], [ enable_asan=no ])
# Define these substitutions here to keep all version information in one place.
# Define these substitions here to keep all version information in one place.
# For information on how to properly maintain the library version information,
# refer to the libtool manual, section "Updating library version information":
# https://www.gnu.org/software/libtool/manual/html_node/Updating-version-info.html
# http://www.gnu.org/software/libtool/manual/html_node/Updating-version-info.html
#
# 1. Start with version information of 0:0:0 for each libtool library.
# 2. Update the version information only immediately before a public release of your software. More frequent updates are unnecessary, and only guarantee that the current interface number gets larger faster.
@ -127,25 +76,25 @@ AC_ARG_ENABLE([asan],
# 4. If any interfaces have been added, removed, or changed since the last update, increment current, and set revision to 0.
# 5. If any interfaces have been added since the last public release, then increment age.
# 6. If any existing interfaces have been removed or changed since the last public release, then set age to 0.
AC_SUBST([LIBPSL_SO_VERSION], m4_normalize(m4_include([libtool_version_info.txt])))
AC_SUBST([LIBPSL_SO_VERSION], [2:5:2])
AC_SUBST([LIBPSL_VERSION], $VERSION)
# Check for enable/disable runtime PSL data
# Check for enable/disable builtin PSL data
AC_ARG_ENABLE(runtime,
[
--enable-runtime[[=IDNA library]]
Specify the IDNA library used for libpsl run-time conversions:
libidn2 [[default]]: IDNA2008 library (also needs libunistring)
libicu: IDNA2008 UTS#46 library
libidn: IDNA2003 library (also needs libunistring)
--disable-runtime Do not link runtime IDNA functionality
libicu [[default]]: IDNA2008 UTS#46 library
libidn2: IDNA2008 library (also needs libunistring)
libidn: IDNA2003 library (also needs libunistring)
--disable-runtime Do not link runtime IDNA functionality
], [
if test "$enableval" = "libidn2" -o "$enableval" = "yes"; then
enable_runtime=libidn2
AC_DEFINE([WITH_LIBIDN2], [1], [generate PSL data using libidn2])
elif test "$enableval" = "libicu"; then
if test "$enableval" = "libicu" -o "$enableval" = "yes"; then
enable_runtime=libicu
AC_DEFINE([WITH_LIBICU], [1], [generate PSL data using libicu])
elif test "$enableval" = "libidn2"; then
enable_runtime=libidn2
AC_DEFINE([WITH_LIBIDN2], [1], [generate PSL data using libidn2])
elif test "$enableval" = "libidn"; then
enable_runtime=libidn
AC_DEFINE([WITH_LIBIDN], [1], [generate PSL data using libidn])
@ -156,20 +105,29 @@ AC_ARG_ENABLE(runtime,
fi
], [
# this is the default if neither --enable-runtime nor --disable-runtime were specified
enable_runtime=auto
enable_runtime=libicu
AC_DEFINE([WITH_LIBICU], [1], [generate PSL data using libicu])
])
# Check for enable/disable builtin PSL data
AC_ARG_ENABLE(builtin,
[
--enable-builtin Generate built-in PSL data
--disable-builtin Do not generate built-in PSL data
--enable-builtin[[=IDNA library]]
Specify the IDNA library used for built-in data generation:
libicu [[default]]: IDNA2008 UTS#46 library
libidn2: IDNA2008 library (also needs libunistring)
libidn: IDNA2003 library (also needs libunistring)
--disable-builtin Do not generate built-in PSL data
], [
if test "$enableval" = "yes"; then
enable_builtin=yes
elif test "$enableval" = "libidn" -o "$enableval" = "libidn2" -o "$enableval" = "libicu"; then
AC_MSG_WARN([--enable-builtin=$enableval is deprecated, use --enable-builtin (enabled by default)])
enable_builtin=yes
if test "$enableval" = "libicu" -o "$enableval" = "yes"; then
enable_builtin=libicu
AC_DEFINE([BUILTIN_GENERATOR_LIBICU], [1], [generate PSL data using libicu])
elif test "$enableval" = "libidn2"; then
enable_builtin=libidn2
AC_DEFINE([BUILTIN_GENERATOR_LIBIDN2], [1], [generate PSL data using libidn2])
elif test "$enableval" = "libidn"; then
enable_builtin=libidn
AC_DEFINE([BUILTIN_GENERATOR_LIBIDN], [1], [generate PSL data using libidn])
elif test "$enableval" = "no"; then
enable_builtin=no
else
@ -177,48 +135,18 @@ AC_ARG_ENABLE(builtin,
fi
], [
# this is the default if neither --enable-builtin nor --disable-builtin were specified
enable_builtin=yes
enable_builtin=libicu
AC_DEFINE([BUILTIN_GENERATOR_LIBICU], [1], [generate PSL data using libicu])
])
if test "$enable_builtin" = "yes"; then
AC_DEFINE([ENABLE_BUILTIN], [1], [Generate built-in PSL data])
fi
if test "$enable_runtime" = "libidn2" -o "$enable_runtime" = "auto"; then
# Check for libidn2
PKG_CHECK_MODULES([LIBIDN2], [libidn2], [
HAVE_LIBIDN2=yes
if test "$enable_runtime" = "libidn2" -o "$enable_runtime" = "auto"; then
LIBS="$LIBIDN2_LIBS $LIBS"
CFLAGS="$LIBIDN2_CFLAGS $CFLAGS"
fi
], [
AC_SEARCH_LIBS(idn2_lookup_u8, idn2, HAVE_LIBIDN2=yes,
[
if test "$enable_runtime" = "libidn2"; then
AC_MSG_ERROR(You requested libidn2 but it is not installed.)
fi
], -lunistring)
])
if test "x$HAVE_LIBIDN2" = "xyes"; then
if test "$enable_runtime" = "auto"; then
enable_runtime=libidn2
AC_DEFINE([WITH_LIBIDN2], [1], [generate PSL data using libidn2])
fi
fi
fi
if test "$enable_runtime" = "libicu" -o "$enable_runtime" = "auto"; then
if test "$enable_runtime" = "libicu" -o "$enable_builtin" = "libicu"; then
# Check for libicu
# using pkg-config won't work on older systems like Ubuntu 12.04 LTS Server Edition 64bit
# using AC_SEARCH_LIBS also don't work since functions have the library version appended
PKG_CHECK_MODULES([LIBICU], [icu-uc], [
HAVE_LIBICU=yes
if test "$enable_runtime" = "libicu" -o "$enable_runtime" = "auto"; then
LIBS="$LIBICU_LIBS $LIBS"
CFLAGS="$LIBICU_CFLAGS $CFLAGS"
fi
LIBS="$LIBICU_LIBS $LIBS"
CFLAGS="$LIBICU_CFLAGS $CFLAGS"
], [
OLDLIBS=$LIBS
LIBS="-licuuc $LIBS"
@ -227,56 +155,31 @@ if test "$enable_runtime" = "libicu" -o "$enable_runtime" = "auto"; then
[AC_LANG_PROGRAM(
[[#include <unicode/ustring.h>]],
[[u_strToUTF8(NULL, 0, NULL, NULL, 0, NULL);]])],
[HAVE_LIBICU=yes; LIBICU_LIBS="-licuuc"; AC_MSG_RESULT([yes])],
[ AC_MSG_RESULT([no]);
if test "$enable_runtime" = "libicu"; then
AC_MSG_ERROR(You requested libicu but it is not installed.)
fi
LIBS=$OLDLIBS
])
[HAVE_LIBICU=yes; AC_MSG_RESULT([yes])],
[AC_MSG_RESULT([no]); AC_MSG_ERROR(You requested libicu but it is not installed.)])
LIBS=$OLDLIBS
])
if test "x$HAVE_LIBICU" = "xyes"; then
if test "$enable_runtime" = "auto"; then
enable_runtime=libicu
AC_DEFINE([WITH_LIBICU], [1], [generate PSL data using libicu])
fi
fi
fi
if test "$enable_runtime" = "libidn" -o "$enable_runtime" = "auto"; then
if test "$enable_runtime" = "libidn2" -o "$enable_builtin" = "libidn2"; then
# Check for libidn2
OLDLIBS=$LIBS
AC_SEARCH_LIBS(idn2_lookup_u8, idn2, HAVE_LIBIDN2=yes, AC_MSG_ERROR(You requested libidn2 but it is not installed.))
LIBS=$OLDLIBS
fi
if test "$enable_runtime" = "libidn" -o "$enable_builtin" = "libidn"; then
# Check for libidn
PKG_CHECK_MODULES([LIBIDN], [libidn], [
HAVE_LIBIDN=yes
if test "$enable_runtime" = "libidn" -o "$enable_runtime" = "auto"; then
LIBS="$LIBIDN_LIBS $LIBS"
CFLAGS="$LIBIDN_CFLAGS $CFLAGS"
fi
], [
AC_SEARCH_LIBS(idna_to_ascii_8z, idn, HAVE_LIBIDN=yes,
[
if test "$enable_runtime" = "libidn"; then
AC_MSG_ERROR(You requested libidn but it is not installed.)
fi
])
])
if test "x$HAVE_LIBIDN" = "xyes"; then
if test "$enable_runtime" = "auto"; then
enable_runtime=libidn
AC_DEFINE([WITH_LIBIDN], [1], [generate PSL data using libidn])
fi
fi
fi
# last fallback is noruntime/nobuiltin
if test "$enable_runtime" = "auto"; then
enable_runtime=no
OLDLIBS=$LIBS
AC_SEARCH_LIBS(idna_to_ascii_8z, idn, HAVE_LIBIDN=yes, AC_MSG_ERROR(You requested libidn but it is not installed.))
LIBS=$OLDLIBS
fi
if test "x$HAVE_LIBIDN2" = "xyes" -o "x$HAVE_LIBIDN" = "xyes"; then
# Check for libunistring, we need it for psl_str_to_utf8lower()
AC_SEARCH_LIBS(u8_tolower, unistring, HAVE_UNISTRING=yes, AC_MSG_ERROR(You requested libidn2|libidn but libunistring is not installed.))
OLDLIBS=$LIBS
AC_SEARCH_LIBS(u8_tolower, unistring, HAVE_UNISTRING=yes, AC_MSG_ERROR(You requested libidn2 but libunistring is not installed.))
LIBS=$OLDLIBS
fi
# AM_ICONV sets @LIBICONV@ and @LTLIBICONV@ for use in Makefile.am
@ -286,7 +189,10 @@ AM_ICONV
AM_CONDITIONAL([WITH_LIBICU], test "x$enable_runtime" = "xlibicu")
AM_CONDITIONAL([WITH_LIBIDN2], test "x$enable_runtime" = "xlibidn2")
AM_CONDITIONAL([WITH_LIBIDN], test "x$enable_runtime" = "xlibidn")
AM_CONDITIONAL([ENABLE_BUILTIN], test "x$enable_builtin" = "xyes")
AM_CONDITIONAL([BUILTIN_GENERATOR_LIBICU], test "x$enable_builtin" = "xlibicu")
AM_CONDITIONAL([BUILTIN_GENERATOR_LIBIDN2], test "x$enable_builtin" = "xlibidn2")
AM_CONDITIONAL([BUILTIN_GENERATOR_LIBIDN], test "x$enable_builtin" = "xlibidn")
AM_CONDITIONAL([WITH_BUILTIN], test $enable_builtin = yes)
# Solaris has socket in libsocket and inet_ntop in libnsl, but also needs libsocket, so the order is important here
AC_CHECK_LIB([socket], [socket], [NEEDS_SOCKET=yes], [])
@ -299,24 +205,8 @@ elif test -n "$NEEDS_SOCKET" ; then
LIBS="$LIBS -lsocket"
elif test -n "$NEEDS_NSL" ; then
LIBS="$LIBS -lnsl"
else
# Platform dependant options
case "${host_os}" in
# MinGW / Windows
*mingw*)
# Select Windows NT/2000 and later, for WSAStringToAddressW()
CPPFLAGS="$CPPFLAGS -D_WIN32_WINNT=0x500"
# Needed for network support
LIBS="$LIBS -lws2_32"
;;
*)
;;
esac
fi
# Check for clock_gettime() used for performance measurement
AC_SEARCH_LIBS(clock_gettime, rt)
# Check for valgrind
ac_enable_valgrind=no
AC_ARG_ENABLE(valgrind-tests,
@ -337,34 +227,23 @@ else
TESTS_INFO="Valgrind testing not enabled"
fi
# Check for distribution-wide PSL file
AC_ARG_WITH(psl-distfile,
AS_HELP_STRING([--with-psl-distfile=[PATH]], [path to distribution-wide PSL file]),
PSL_DISTFILE=$withval AC_SUBST(PSL_DISTFILE))
# Check for custom PSL file
AC_ARG_WITH(psl-file,
AS_HELP_STRING([--with-psl-file=[PATH]], [path to PSL file]),
AC_HELP_STRING([--with-psl-file=[PATH]], [path to PSL file]),
PSL_FILE=$withval,
PSL_FILE="\$(top_srcdir)/list/public_suffix_list.dat")
AC_SUBST(PSL_FILE)
# Check for custom PSL test file
AC_ARG_WITH(psl-testfile,
AS_HELP_STRING([--with-psl-testfile=[PATH]], [path to PSL test file]),
AC_HELP_STRING([--with-psl-testfile=[PATH]], [path to PSL test file]),
PSL_TESTFILE=$withval,
PSL_TESTFILE="\$(top_srcdir)/list/tests/tests.txt")
PSL_TESTFILE="\$(top_srcdir)/list/tests/test_psl.txt")
AC_SUBST(PSL_TESTFILE)
# check for alloca / alloca.h
AC_FUNC_ALLOCA
AC_CHECK_FUNCS([strndup clock_gettime fmemopen nl_langinfo])
# check for dirent.h
AC_HEADER_DIRENT
# strings.h may not exist on WIN32
AC_CHECK_HEADERS([strings.h])
AC_CHECK_FUNCS([strndup])
# Override the template file name of the generated .pc file, so that there
# is no need to rename the template file when the API version changes.
@ -373,13 +252,8 @@ AC_CONFIG_FILES([Makefile
src/Makefile
tools/Makefile
po/Makefile.in
fuzz/Makefile
tests/Makefile
docs/libpsl/Makefile docs/libpsl/version.xml
libpsl.pc:libpsl.pc.in
msvc/Makefile
msvc/config.h.win32
msvc/config-msvc.mak])
libpsl.pc:libpsl.pc.in])
AC_OUTPUT
AC_MSG_NOTICE([Summary of build options:
@ -393,12 +267,7 @@ AC_MSG_NOTICE([Summary of build options:
Libs: ${LIBS}
Runtime: ${enable_runtime}
Builtin: ${enable_builtin}
PSL Dist File: ${PSL_DISTFILE}
PSL File: ${PSL_FILE}
PSL Test File: ${PSL_TESTFILE}
Sanitizers: UBSan $enable_ubsan, ASan $enable_asan, CFI $enable_cfi
Docs: $enable_gtk_doc
Man pages: $enable_man
Tests: ${TESTS_INFO}
Fuzzing build: $enable_fuzzing, $LIB_FUZZING_ENGINE
])

View File

@ -6,14 +6,15 @@
set -e
set -u
CORES=$(nproc)
CORES=$(grep -c ^processor /proc/cpuinfo 2> /dev/null || echo 0)
if [ $CORES -lt 1 ]; then CORES=1; fi
echo "Running: make distclean"
make distclean > /dev/null || true
# We define _GNU_SOURCE to avoid warnings with missing prototypes.
# C89 does not know snprintf, strdup, strndup, popen, pclose
CFLAGS="-std=gnu89 -pedantic -g -Wall -Wextra -Wstrict-prototypes -Wold-style-definition -Wwrite-strings -Wshadow -Wformat -Wformat-security -Wunreachable-code -Wstrict-prototypes -Wmissing-prototypes -Wold-style-definition"
CFLAGS="-std=c89 -pedantic -O2 -g -Wall -Wextra -Wstrict-prototypes -Wold-style-definition -Wwrite-strings -Wshadow -Wformat -Wformat-security -Wunreachable-code -Wstrict-prototypes -Wmissing-prototypes -Wold-style-definition -D_GNU_SOURCE"
CACHEFILE=$PWD/config_check.cache
@ -37,52 +38,27 @@ for CC in gcc clang; do
rm -f $CACHEFILE
for options in \
"--enable-runtime=libicu" \
"--enable-runtime=libidn2" \
"--enable-runtime=libidn" \
"--disable-runtime"; do
export DISTCHECK_CONFIGURE_FLAGS="-q -C --cache-file=$CACHEFILE $options"
if test "$CC" = "clang"; then
echo
echo " *** CC=$CC ./configure $DISTCHECK_CONFIGURE_FLAGS --enable-cfi"
./configure $DISTCHECK_CONFIGURE_FLAGS --enable-cfi CFLAGS="$CFLAGS"
for xLCALL in C tr_TR.utf8; do
make -s clean
make -s check -j$CORES
done
fi
for flags in --enable-ubsan --enable-asan; do
# case "$flags" in *-asan) case $options in *libicu*) continue;; esac;; esac
echo
echo " *** CC=$CC ./configure $DISTCHECK_CONFIGURE_FLAGS $flags"
./configure $DISTCHECK_CONFIGURE_FLAGS $flags CFLAGS="$CFLAGS"
for xLCALL in C tr_TR.utf8; do
make -s clean
make -s check -j$CORES
done
done
"--enable-runtime=libicu --enable-builtin=libicu" \
"--enable-runtime=libidn2 --enable-builtin=libidn2" \
"--enable-runtime=libidn --enable-builtin=libidn"; do
export DISTCHECK_CONFIGURE_FLAGS="-C --cache-file=$CACHEFILE $options"
echo
echo " *** CC=$CC ./configure $DISTCHECK_CONFIGURE_FLAGS"
./configure $DISTCHECK_CONFIGURE_FLAGS CFLAGS="$CFLAGS"
echo " *** ./configure $DISTCHECK_CONFIGURE_FLAGS"
./configure $DISTCHECK_CONFIGURE_FLAGS CFLAGS="$CFLAGS" > /dev/null
for xVALGRIND in 0 1; do
for xLCALL in C tr_TR.utf8; do
export TESTS_ENVIRONMENT="LC_ALL=$xLCALL VALGRIND_TESTS=$xVALGRIND"
echo " *** TESTS_ENVIRONMENT=\"$TESTS_ENVIRONMENT\"" make check -j$CORES
make -s clean
make -s check -j$CORES
make check -j$CORES > /dev/null
done
done
unset TESTS_ENVIRONMENT
export TESTS_ENVIRONMENT
echo
echo " *** $CC: make distcheck CFLAGS=$CFLAGS -j$CORES"
./configure $DISTCHECK_CONFIGURE_FLAGS CFLAGS="$CFLAGS"
make -s clean
make -s check -j$CORES
make -s distcheck CFLAGS="$CFLAGS" -j$CORES
echo " *** make distcheck CFLAGS=$CFLAGS -j$CORES"
make distcheck CFLAGS="$CFLAGS" -j$CORES > /dev/null
done
done

View File

@ -1,69 +0,0 @@
#!/bin/sh
#
# Do some checking before 'git push'.
# Set a stricter bash mode
set -e
set -u
# We define _GNU_SOURCE to avoid warnings with missing prototypes.
# C89 does not know snprintf, strdup, strndup, popen, pclose
export CFLAGS="-std=gnu89 -pedantic -g -Wall -Wextra -Wstrict-prototypes -Wold-style-definition -Wwrite-strings -Wshadow -Wformat -Wformat-security -Wunreachable-code -Wstrict-prototypes -Wmissing-prototypes -Wold-style-definition"
ASAN_FLAGS="-fsanitize=address -fno-omit-frame-pointer -fsanitize-address-use-after-scope"
# measure time consumed and print it at the end of the script
START=$(date +%s.%N)
for CC in clang gcc; do
export CC
echo
echo "*** Testing with CC=$CC"
for options in \
"-Dbuiltin=libicu -Druntime=libicu" \
"-Dbuiltin=libidn2 -Druntime=libidn2" \
"-Dbuiltin=libidn -Druntime=libidn" \
"-Dbuiltin=no -Druntime=no"; do
# normal build+check+valgrind-check
echo
echo " *** $options"
rm -rf builddir
meson builddir $options
cd builddir
ninja
for xLCALL in C tr_TR.utf8; do
LC_ALL=$xLCALL meson test
LC_ALL=$xLCALL meson test --wrap='valgrind --tool=memcheck'
done
cd ..
# UBSAN build+check
flags="-Db_sanitize=undefined -Db_lundef=false"
echo
echo " *** $options $flags"
rm -rf builddir
LDFLAGS="-lubsan" meson builddir $options $flags
LDFLAGS="-lubsan" ninja -C builddir
for xLCALL in C tr_TR.utf8; do
LC_ALL=$xLCALL ninja test -C builddir
done
# ASAN build+check
flags="-Db_sanitize=address -Db_lundef=false"
echo
echo " *** $options $flags"
rm -rf builddir
CFLAGS="$CFLAGS $ASAN_FLAGS" meson builddir $options $flags
CFLAGS="$CFLAGS $ASAN_FLAGS" ninja -C builddir
for xLCALL in C tr_TR.utf8; do
LC_ALL=$xLCALL CFLAGS="$CFLAGS $ASAN_FLAGS" ninja test -C builddir
done
done
done
END=$(date +%s.%N)
echo
echo "Duration: "$(echo "$END - $START" | bc)

View File

@ -1,57 +0,0 @@
#!/bin/bash
set -e
# just in case...
unset CC
unset CXX
#PREFIX=i686-w64-mingw32
PREFIX=x86_64-w64-mingw32
#export PATH="/usr/$PREFIX/bin:$PATH"
export INSTALLDIR="$PWD/$PREFIX"
export PKG_CONFIG_PATH=$INSTALLDIR/lib/pkgconfig:/usr/$PREFIX/lib/pkgconfig
export CPPFLAGS="-I$INSTALLDIR/include"
export LDFLAGS="-L$INSTALLDIR/lib"
# let mingw compiler be less verbose
export CFLAGS="-O2 -Wall -Wno-format"
# build libunistring
if [ ! -d libunistring ]; then
git clone https://git.savannah.gnu.org/git/libunistring.git
cd libunistring
else
cd libunistring
git pull
fi
./autogen.sh
./configure --build=x86_64-pc-linux-gnu --host=$PREFIX --enable-shared --prefix=$INSTALLDIR
make clean
make -j$(nproc)
make install
cd ..
# build libidn2
if [ ! -d libidn2 ]; then
git clone https://gitlab.com/libidn/libidn2.git
cd libidn2
else
cd libidn2
git pull
fi
./bootstrap
./configure --build=x86_64-pc-linux-gnu --host=$PREFIX --enable-shared --disable-doc --disable-gcc-warnings --prefix=$INSTALLDIR
make clean
make -j$(nproc)
make install
cd ..
# build libpsl
export WINEPATH="$INSTALLDIR/bin;$INSTALLDIR/lib;/usr/$PREFIX/bin;/usr/$PREFIX/lib;$PWD/libpsl/.libs;$GCCLIB"
./autogen.sh
./configure --build=x86_64-pc-linux-gnu --host=$PREFIX --enable-shared --disable-gtk-doc --enable-runtime=libidn2 --prefix=$INSTALLDIR
make clean
make -j$(nproc)
make check -j$(nproc) LOG_COMPILER=wine

View File

@ -1,9 +0,0 @@
#!/usr/bin/env sh
#
# see https://github.com/codespell-project/codespell
#
# in short:
# - pip install codespell
# - add ~/.local/bin to PATH
codespell $* -L "tim" `git ls-files|egrep -v '_fuzzer.in|_fuzzer.repro|^list$'`

View File

@ -68,7 +68,7 @@ HTML_IMAGES=
# e.g. content_files=running.sgml building.sgml changes-2.0.sgml
content_files=
# SGML files where gtk-doc abbreviations (#GtkWidget) are expanded
# SGML files where gtk-doc abbrevations (#GtkWidget) are expanded
# These files must be listed here *and* in content_files
# e.g. expand_content_files=running.sgml
expand_content_files=
@ -86,7 +86,7 @@ include $(top_srcdir)/gtk-doc.make
# Other files to distribute
# e.g. EXTRA_DIST += version.xml.in
EXTRA_DIST += meson.build
EXTRA_DIST +=
# Files not to distribute
# for --rebuild-types in $(SCAN_OPTIONS), e.g. $(DOC_MODULE).types

View File

@ -11,7 +11,7 @@
<releaseinfo>
for Libpsl &version;.
The latest version of this documentation can be found on-line at
<ulink role="online-location" url="https://github.com/rockdaboot/libpsl">GitHub</ulink>.
<ulink role="online-location" url="http://github.com/rockdaboot/libpsl">GitHub</ulink>.
</releaseinfo>
</bookinfo>
@ -22,6 +22,10 @@
</para>
<xi:include href="xml/libpsl.xml"/>
</chapter>
<chapter id="object-tree">
<title>Object Hierarchy</title>
<xi:include href="xml/tree_index.sgml"/>
</chapter>
<index id="api-index-full">
<title>API Index</title>
<xi:include href="xml/api-index-full.xml"><xi:fallback /></xi:include>
@ -30,35 +34,6 @@
<title>Index of deprecated API</title>
<xi:include href="xml/api-index-deprecated.xml"><xi:fallback /></xi:include>
</index>
<index role="0.1">
<title>Index of symbols included in 0.1</title>
<xi:include href="xml/api-index-0.1.xml"><xi:fallback/></xi:include>
</index>
<index role="0.2.5">
<title>New symbols included in 0.2.5</title>
<xi:include href="xml/api-index-0.2.5.xml"><xi:fallback/></xi:include>
</index>
<index role="0.4">
<title>New symbols included in 0.4</title>
<xi:include href="xml/api-index-0.4.xml"><xi:fallback/></xi:include>
</index>
<index role="0.10.0">
<title>New symbols included in 0.10.0</title>
<xi:include href="xml/api-index-0.10.0.xml"><xi:fallback/></xi:include>
</index>
<index role="0.11.0">
<title>Index of symbols included in 0.11.0</title>
<xi:include href="xml/api-index-0.11.0.xml"><xi:fallback/></xi:include>
</index>
<index role="0.16">
<title>Index of symbols included in 0.16</title>
<xi:include href="xml/api-index-0.16.xml"><xi:fallback/></xi:include>
</index>
<index role="0.19">
<title>Index of symbols included in 0.19</title>
<xi:include href="xml/api-index-0.19.xml"><xi:fallback/></xi:include>
</index>
<!--
<xi:include href="xml/annotation-glossary.xml"><xi:fallback /></xi:include>

View File

@ -1,38 +1,22 @@
<SECTION>
<FILE>libpsl</FILE>
<TITLE>Public Suffix List functions</TITLE>
PSL_API
PSL_VERSION
PSL_VERSION_MAJOR
PSL_VERSION_MINOR
PSL_VERSION_NUMBER
PSL_VERSION_PATCH
PSL_TYPE_ICANN
PSL_TYPE_PRIVATE
PSL_TYPE_NO_STAR_RULE
PSL_TYPE_ANY
psl_error_t
psl_ctx_t
psl_load_file
psl_load_fp
psl_latest
psl_builtin
psl_free
psl_is_public_suffix
psl_is_public_suffix2
psl_unregistrable_domain
psl_registrable_domain
psl_suffix_count
psl_suffix_exception_count
psl_suffix_wildcard_count
psl_builtin_compile_time
psl_builtin_file_time
psl_builtin_sha1sum
psl_builtin_filename
psl_builtin_outdated
psl_is_cookie_domain_acceptable
psl_dist_filename
psl_get_version
psl_check_version_number
psl_str_to_utf8lower
psl_free_string
</SECTION>

View File

@ -1,20 +0,0 @@
gnome = import('gnome')
if get_option('docs')
configure_file(
input : 'version.xml.in',
output : 'version.xml',
configuration : cdata)
gnome.gtkdoc('libpsl',
main_sgml: 'libpsl-docs.sgml',
src_dir: [
'src',
'include',
],
dependencies: libpsl,
scan_args: [
'--ignore-decorators=G_GNUC_PSL_UNUSED',
],
install: true)
endif

View File

@ -1,99 +0,0 @@
AM_CFLAGS = $(WERROR_CFLAGS) $(WARN_CFLAGS)
AM_CPPFLAGS = -I$(top_srcdir)/include -I$(srcdir) -DSRCDIR=\"$(abs_srcdir)\"
#AM_LDFLAGS = -static
LDADD = ../src/libpsl.la $(LIBICU_LIBS) $(LIBIDN_LIBS) $(LIBIDN2_LIBS)
if !FUZZING
MAIN = main.c fuzzer.h
endif
if WITH_LIBICU
XLIBS = $(LIBICU_LIBS)
XTYPE = _icu
libpsl_icu_fuzzer_SOURCES = libpsl_fuzzer.c $(MAIN)
libpsl_icu_load_fuzzer_SOURCES = libpsl_load_fuzzer.c $(MAIN)
libpsl_icu_load_dafsa_fuzzer_SOURCES = libpsl_load_dafsa_fuzzer.c $(MAIN)
PSL_TESTS = \
libpsl_icu_fuzzer$(EXEEXT) \
libpsl_icu_load_fuzzer$(EXEEXT) \
libpsl_icu_load_dafsa_fuzzer$(EXEEXT)
else
if WITH_LIBIDN2
XLIBS = -lidn2 -lunistring
XTYPE = _idn2
libpsl_idn2_fuzzer_SOURCES = libpsl_fuzzer.c $(MAIN)
libpsl_idn2_load_fuzzer_SOURCES = libpsl_load_fuzzer.c $(MAIN)
libpsl_idn2_load_dafsa_fuzzer_SOURCES = libpsl_load_dafsa_fuzzer.c $(MAIN)
PSL_TESTS = \
libpsl_idn2_fuzzer$(EXEEXT) \
libpsl_idn2_load_fuzzer$(EXEEXT) \
libpsl_idn2_load_dafsa_fuzzer$(EXEEXT)
else
if WITH_LIBIDN
XLIBS = -lidn -lunistring
XTYPE = _idn
libpsl_idn_fuzzer_SOURCES = libpsl_fuzzer.c $(MAIN)
libpsl_idn_load_fuzzer_SOURCES = libpsl_load_fuzzer.c $(MAIN)
libpsl_idn_load_dafsa_fuzzer_SOURCES = libpsl_load_dafsa_fuzzer.c $(MAIN)
PSL_TESTS = \
libpsl_idn_fuzzer$(EXEEXT) \
libpsl_idn_load_fuzzer$(EXEEXT) \
libpsl_idn_load_dafsa_fuzzer$(EXEEXT)
else
XLIBS =
XTYPE =
libpsl_fuzzer_SOURCES = libpsl_fuzzer.c $(MAIN)
libpsl_load_fuzzer_SOURCES = libpsl_load_fuzzer.c $(MAIN)
libpsl_load_dafsa_fuzzer_SOURCES = libpsl_load_dafsa_fuzzer.c $(MAIN)
PSL_TESTS = \
libpsl_fuzzer$(EXEEXT) \
libpsl_load_fuzzer$(EXEEXT) \
libpsl_load_dafsa_fuzzer$(EXEEXT)
endif
endif
endif
if FUZZING
bin_PROGRAMS = $(PSL_TESTS)
LDADD += $(LIB_FUZZING_ENGINE)
AM_LDFLAGS = -no-install
else
AM_CPPFLAGS += -DTEST_RUN
AM_TESTS_ENVIRONMENT = export VALGRIND_TESTS"=@VALGRIND_TESTS@";
TESTS = $(PSL_TESTS)
check_PROGRAMS = $(PSL_TESTS)
endif
EXTRA_DIST = meson.build
dist-hook:
find . -name '*.options' -exec cp -v '{}' $(distdir) ';'
find . -name '*.dict' -exec cp -v '{}' $(distdir) ';'
find . -name '*.in' -exec cp -vr '{}' $(distdir) ';'
find . -name '*.repro' -exec cp -vr '{}' $(distdir) ';'
TESTS_ENVIRONMENT = TESTS_VALGRIND="@VALGRIND_ENVIRONMENT@"
clean-local:
rm -rf *.gc?? *.log lcov coverage.info *_fuzzer *.o
fuzz-coverage: $(PSL_TESTS)
find . -name '*_fuzzer' -exec ./coverage.sh '{}' ';'
CXX ?= clang
CXXFLAGS ?= $(CFLAGS)
oss-fuzz:
if test "$$OUT" != ""; then \
for cfile in `ls *_fuzzer.c`; do \
fuzzer="libpsl$(XTYPE)_"$$(basename $$cfile .c|cut -d'_' -f2-); \
$$CXX $$CXXFLAGS $$CPPFLAGS -I$(top_srcdir)/include -I$(top_srcdir) \
"$${cfile}" -o "$${fuzzer}" \
../src/.libs/libpsl.a $${LIB_FUZZING_ENGINE} \
-Wl,-Bstatic \
$(XLIBS) \
-Wl,-Bdynamic; \
done; \
fi
.PHONY: oss-fuzz

View File

@ -1,108 +0,0 @@
# Fuzzers
These are fuzzers designed for use with `libFuzzer` or `afl`. They can
be used to run on Google's OSS-Fuzz (https://github.com/google/oss-fuzz/).
The convention used here is that the initial values for each parser fuzzer
are taken from the $NAME.in directory.
Crash reproducers from OSS-Fuzz are put into $NAME.repro directory for
regression testing with top dir 'make check' or 'make check-valgrind'.
The ./configure runs below are for libidn2.
To test libicu replace 'libidn2' with 'libicu', to test with
libidn replace 'libidn2' by 'libidn'.
To test without IDNA libraries replace `--enable-runtime=...` with `--disable-runtime`
and replace `--enable-builtin=...` with `--disable-builtin`.
# Running a fuzzer using clang
Use the following commands on top dir:
```
export CC=clang
export LIB_FUZZING_ENGINE="-lFuzzer -lstdc++"
export UBSAN_OPTIONS=print_stacktrace=1
export ASAN_SYMBOLIZER_PATH=/usr/bin/llvm-symbolizer
# set flags for clang asan
CFLAGS="-O1 -fno-omit-frame-pointer -gline-tables-only -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=address,leak,nonnull-attribute -fsanitize-address-use-after-scope -fsanitize-coverage=trace-pc-guard,trace-cmp"
# set flags for clang ubsan
CFLAGS="$CFLAGS -fsanitize=bool,array-bounds,float-divide-by-zero,function,integer-divide-by-zero,return,shift,signed-integer-overflow,vla-bound,vptr,undefined,alignment,null,enum,integer,builtin,float-divide-by-zero,function,object-size,returns-nonnull-attribute,unsigned-integer-overflow,unreachable -fsanitize=fuzzer-no-link"
# unsigned-integer-overflow is not UB, so recover when we see it.
CFLAGS="$CFLAGS -fno-sanitize-recover=all -fsanitize-recover=unsigned-integer-overflow"
export CFLAGS
./configure --enable-static --disable-gtk-doc --enable-fuzzing --enable-runtime=libidn2 --enable-builtin=libidn2
make -j$(nproc)
cd fuzz
# run libpsl_fuzzer
./run-clang.sh libpsl_fuzzer
```
If you see a crash, then a crash corpora is written that can be used for further
investigation. E.g.
```
==2410==ERROR: AddressSanitizer: heap-use-after-free on address 0x602000004e90 at pc 0x00000049cf9c bp 0x7fffb5543f70 sp 0x7ff
fb5543720
...
Test unit written to ./crash-adc83b19e793491b1c6ea0fd8b46cd9f32e592fc
```
To reproduce the crash:
```
./libpsl_fuzzer < ./crash-adc83b19e793491b1c6ea0fd8b46cd9f32e592fc
```
You can also copy/move that file into libpsl_fuzzer.repro/
and re-build the project without fuzzing for a valgrind run, if you like that better.
Just a `./configure` and a `make check-valgrind` should reproduce it.
# Running a fuzzer using AFL
Use the following commands on top dir:
```
$ CC=afl-clang-fast ./configure --disable-gtk-doc --enable-runtime=libidn2 --enable-builtin=libidn2
$ make -j$(nproc) clean all
$ cd fuzz
$ ./run-afl.sh libpsl_fuzzer
```
# Fuzz code coverage using the corpus directories *.in/
Code coverage reports currently work best with gcc+lcov+genhtml.
In the top directory:
```
CC=gcc CFLAGS="-O0 -g" ./configure --disable-gtk-doc --enable-runtime=libidn2 --enable-builtin=libidn2
make fuzz-coverage
xdg-open lcov/index.html
```
Each fuzzer target has it's own functions to cover, e.g.
`libpsl_fuzzer` covers psl_is_public_suffix.
To work on corpora for better coverage, `cd fuzz` and use e.g.
`./view-coverage.sh libpsl_fuzzer`.
# Enhancing the testsuite for issues found
Each reproducer file should be dropped into the appropriate *.repro/
directory.
# Clang CFI instrumentation
```
CC=clang CFLAGS="-B/usr/bin/gold -O0 -fsanitize=cfi -flto -fvisibility=default -fno-sanitize-trap=all" ./configure
make clean
make
make check
```

View File

@ -1,5 +0,0 @@
#!/bin/sh
for f in $1.in/*; do
$1 < $f >/dev/null
done

View File

@ -1,38 +0,0 @@
/*
* Copyright(c) 2017-2022 Tim Ruehsen
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*
* This file is part of libpsl.
*/
#include <config.h>
#include <stddef.h> /* size_t */
#ifdef HAVE_STDINT_H
#include <stdint.h> /* uint8_t */
#elif defined (_MSC_VER)
typedef unsigned __int8 uint8_t;
#endif
#ifdef __cplusplus
extern "C"
#endif
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size);

View File

@ -1,18 +0,0 @@
#!/bin/sh -eu
#
#
if ! grep -q FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION Makefile; then
echo "The build environment hasn't been set up for fuzzing (likely for regression testing)."
echo "Please built regarding README.md and try again."
exit 1
fi
if $(grep -q '^#define WITH_LIBIDN2 1' ../config.h); then RUNTIME="_idn2";
elif $(grep -q '^#define WITH_LIBIDN 1' ../config.h); then RUNTIME="_idn";
elif $(grep -q '^#define WITH_LIBICU 1' ../config.h); then RUNTIME="_icu";
else RUNTIME=""; fi
for fuzzer in `ls *_fuzzer.c|cut -d'_' -f2-|cut -d'.' -f1`; do
./get_ossfuzz_corpora libpsl${RUNTIME}_$fuzzer
done

View File

@ -1,46 +0,0 @@
#!/bin/sh -eu
# As a first step see README.md and follow the steps under "Running a fuzzer using clang".
# You might need 'gsutil' to download new corpora from the Google cloud:
# Read the docs at https://github.com/google/oss-fuzz/blob/master/docs/corpora.md
# then install 'google-cloud-sdk' and execute 'gcloud init'.
# Now 'gsutil' should be ready to use.
if test -z "$1"; then
echo "Usage: $0 <fuzzer target>"
echo "Example: $0 libpsl_fuzzer"
exit 1
fi
if ! grep -q FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION Makefile; then
echo "The fuzzers haven't been built for fuzzing (maybe for regression testing !?)"
echo "Please built regarding README.md and try again."
exit 1
fi
fuzzer=$1
project=$(echo $1 | cut -d'_' -f1)
# sync/copy the OSS-Fuzz corpora into the .new directory
mkdir -p ${fuzzer}.in ${fuzzer}.new
cp -fp ${fuzzer}.in/* ${fuzzer}.new 2>/dev/null || true
gsutil cp $(gsutil ls gs://${project}-backup.clusterfuzz-external.appspot.com/corpus/libFuzzer/${fuzzer}|tail -n 1) ${fuzzer}.new/
(cd ${fuzzer}.new && unzip -o *.zip && mv *.zip ..)
# create fuzzer target
BUILD_ONLY=1 ./run-clang.sh ${fuzzer}
# merge the corpora into the .in directory
./${fuzzer} -merge=1 ${fuzzer}.in ${fuzzer}.new
# now clear .new dir and put all corpora there
rm -rf ${fuzzer}.new
mv ${fuzzer}.in ${fuzzer}.new
mkdir ${fuzzer}.in
# now merge again (optimizes number of corpora)
./${fuzzer} -merge=1 ${fuzzer}.in ${fuzzer}.new
echo
echo "If new files have been added, 'git add' and 'git commit' them."

View File

@ -1,103 +0,0 @@
/*
* Copyright(c) 2017-2022 Tim Ruehsen
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*
* This file is part of libpsl.
*/
#include <config.h>
#include <assert.h> /* assert */
#ifdef HAVE_STDINT_H
#include <stdint.h> /* uint8_t */
#elif defined (_MSC_VER)
typedef unsigned __int8 uint8_t;
#endif
#include <stdlib.h> /* malloc, free */
#include <string.h> /* memcpy */
#if defined(WITH_LIBICU)
#include <unicode/uclean.h>
#endif
#include "libpsl.h"
#include "fuzzer.h"
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
{
static int first_run = 1;
psl_ctx_t *psl;
char *domain, *res;
int rc;
if (size > 64 * 1024 - 1)
return 0;
domain = (char *) malloc(size + 1);
assert(domain != NULL);
/* 0 terminate */
memcpy(domain, data, size);
domain[size] = 0;
psl = (psl_ctx_t *) psl_builtin();
psl_is_public_suffix(psl, domain);
psl_is_public_suffix2(psl, domain, PSL_TYPE_PRIVATE);
psl_is_public_suffix2(psl, domain, PSL_TYPE_ICANN);
psl_is_public_suffix2(psl, domain, PSL_TYPE_NO_STAR_RULE);
psl_is_public_suffix2(psl, domain, PSL_TYPE_NO_STAR_RULE|PSL_TYPE_ANY);
psl_unregistrable_domain(psl, domain);
psl_registrable_domain(psl, domain);
psl_is_cookie_domain_acceptable(psl, "", NULL);
psl_is_cookie_domain_acceptable(psl, "a.b.c.e.com", domain);
if ((rc = psl_str_to_utf8lower(domain, "utf-8", NULL, &res)) == PSL_SUCCESS)
free(res);
if ((rc = psl_str_to_utf8lower(domain, "iso-8859-1", NULL, &res)) == PSL_SUCCESS)
free(res);
if ((rc = psl_str_to_utf8lower(domain, NULL, NULL, &res)) == PSL_SUCCESS)
free(res);
psl_free(psl);
if (first_run) {
psl_is_public_suffix(NULL, domain);
psl_check_version_number(1);
psl_get_version();
psl_dist_filename();
psl_builtin_outdated();
psl_builtin_filename();
psl_builtin_sha1sum();
psl_builtin_file_time();
first_run = 0;
}
free(domain);
#if defined(WITH_LIBICU)
u_cleanup(); /* free all library internal memory to avoid memory leaks being reported */
#endif
return 0;
}

View File

@ -1 +0,0 @@
......................................................................mmmmmmmmmmm.........................................................................................

View File

@ -1 +0,0 @@
...........................................................................................................................................................................

View File

@ -1 +0,0 @@
XBBBBBBBBBBBBBBBB.BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBxc}m

View File

@ -1 +0,0 @@
............................................

View File

@ -1 +0,0 @@
......................................................................................

View File

@ -1,2 +0,0 @@
IIIII
IIIIIIIIIIIIIIIIIIIIIIIIIIIIIKIIIIIIIIIIIIIIIIIIIKIIIIIIIIAIIIIIIIIIIIIIIIIIIIIII

View File

@ -1 +0,0 @@
X.BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBxcom

View File

@ -1 +0,0 @@
X:X:ccoEEEEEEEEm

View File

@ -1 +0,0 @@
;.VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV

View File

@ -1 +0,0 @@
.............................................................................................................-...................

View File

@ -1 +0,0 @@
mm.mm.mm.mm.mm.mm.mm.mm.mm.mm.mm.mm.mm.mm.mm.mm.

View File

@ -1 +0,0 @@
l.cc.co.usc...cc.co.usc....co.us<..co.us<

View File

@ -1 +0,0 @@
cc.co.us<63>.s.se

View File

@ -1 +0,0 @@
6.6..в.4..в.в.

View File

@ -1 +0,0 @@
lZź...Z..źÎZ.ĂlddŇŃŃŃŇŃŃ

View File

@ -1 +0,0 @@
酶.酶.酶.<2E>

View File

@ -1 +0,0 @@
.............................................................................................................................e.com

View File

@ -1 +0,0 @@
......................................................................mmmmmmmmmmm.........................................................................................

View File

@ -1 +0,0 @@
в≥в╗в.в≥в╗в..в≥в╗в..в≥в╗в.в≥вв╗в..в≥в╗в.в≥в╗в..в≥в╗в.в╗в..в≥в╗в.в≥в╗в.

View File

@ -1 +0,0 @@
.............................e.com

View File

@ -1 +0,0 @@
6.6.6.в.4..в.4..в.в.6.6.в.4..в.4..в.в.6.в.4..в.4..4..в.4..в.вв.в.

View File

@ -1 +0,0 @@
.в.в≥в.co.keFЪЪX:X:ccoм.dЪIЪУ.dЪм.dЪIЪУ.dЪЪЪ╓ц╤*.ц╓Ю

View File

@ -1 +0,0 @@
...........................................................................................................................................................................

View File

@ -1 +0,0 @@
x.x..x.c..*.x.x..x..x.c...x.c.η.c.η...x.c...x.x..x..x.c...x.c.η.c.η.

View File

@ -1 +0,0 @@
fbxos.fbxos.fbxos.fbxos.fbxos.fbxos.fbxos.fxos.fbxos.fbxos.fbxos.fbxos.fbxos.fbxos.fbxos.fxos.

View File

@ -1 +0,0 @@
XBBBBBBBBBBBBBBBB.BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBxc}m

Some files were not shown because too many files have changed in this diff Show More