Added amiga makefile for creating the releases

Signed-off-by: David A. Wheeler <dwheeler@dwheeler.com>

.github/workflows/main.yml

@@ -0,0 +1,27 @@
+on: [push]
+
+jobs:
+  flawfinder:
+    name: Flawfinder
+    runs-on: ubuntu-latest
+    steps:
+      # To use this repository's private action,
+      # you must check out the repository
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Flawfinder action step
+        uses: ./ # Uses an action in the root directory
+        with:
+          arguments: '--sarif ./setup.py'
+          output: 'flawfinder_results.sarif'
+
+      - name: Upload a Build Artifact
+        uses: actions/upload-artifact@v2.2.4
+        with:
+          path: flawfinder_results.sarif   
+
+      - name: Upload analysis results to GitHub Security tab
+        uses: github/codeql-action/upload-sarif@v1
+        with:
+          sarif_file: ${{github.workspace}}/flawfinder_results.sarif

CONTRIBUTING.md

@@ -62,8 +62,8 @@ As always, if you're modifying the software, you'll need to have
 your development environment set up. You need:
 
 * make
-* python2 (invokable as "python2")
-* python3 (invokable as "python3")
+* python2 (invocable as "python2")
+* python3 (invocable as "python3")
 * pylint (see below)
 
 An easy way to install pylint is to use pip.

ChangeLog

@@ -1,3 +1,39 @@
+2021-08-29 David A. Wheeler
+	* Version 2.0.19
+	* Fix so we send error messages to stderr instead of stdout.
+	  Originally we sent some to stdout by mistake, which could
+	  mess up results since the error messages would be mixed up
+	  with the results.
+
+2021-06-24 David A. Wheeler
+	* Version 2.0.18
+	* Fix SARIF output. SARIF output is new to flawfinder, and
+	  there was a subtle error in its generation that causes GitHub
+	  to reject the SARIF file.
+
+2021-06-02 David A. Wheeler
+	* Version 2.0.17
+	* Fix the distributed tarball, which didn't include the
+	   key source file due to the earlier file restructure.
+	* Minor code style fix, which simplifies the code slightly.
+	* Update date in manual page to 2021. That's important because
+	  the documentation now includes information on `--sarif`.
+
+2021-05-31 David A. Wheeler
+	* Version 2.0.16
+	* The distributed source file is now flawfinder.py, not flawfinder.
+	  This is part of a change that improves
+	  improve cross-platform ease-of-use by using entry_points.
+	  That said, "make install" will still
+	  install it as "flawfinder" (so those who install it via
+	  "make install" will see no change). Many thanks to Ben Spoor!
+	* Added support for generating SARIF output, use --sarif.
+	  A big thanks to Yong Yan for this work!
+	* Track curly brace level to reduce some problems, my thanks to
+	  Greg Myers for the work!
+	* Improved handling of Git patch format, thanks to
+	  Robin Geffroy.
+
 2021-01-11 David A. Wheeler
 	* Version 2.0.15
 	* Fixed some release problems in 2.0.14.

Dockerfile

@@ -0,0 +1,10 @@
+# Container image that runs your code
+FROM python:3
+
+# Copies your code file from your action repository to the filesystem path `/` of the container
+COPY entrypoint.sh /entrypoint.sh
+
+RUN pip install flawfinder
+
+# Code file to execute when the docker container starts up (`entrypoint.sh`)
+ENTRYPOINT ["/entrypoint.sh"]

Makefile.amiga

@@ -0,0 +1,20 @@
+#
+# Project: flawfinder
+# Created by George "walkero" Sokianos
+# 2022-07-25
+#
+
+release: clean
+	mkdir -p release/flawfinder
+	cp -r release_files/* release/flawfinder/
+	cp flawfinder.py release/flawfinder/flawfinder
+	protect release/flawfinder/flawfinder srwed
+	cp -r simplejson release/flawfinder
+	cp ChangeLog release/flawfinder/
+	cp README.md release/flawfinder/
+	cp COPYING release/flawfinder/
+	lha -aeqr3 a flawfinder.lha release/
+
+clean:
+	rm -f simplejson/#?.pyc
+

README.md

@@ -65,7 +65,7 @@ is configured to use UTF-8 (the most common situation by far).
 
 However, it's possible for flawfinder to halt if there is a
 character encoding problem and you're running Python3.
-The usual symptom is error meesages like this:
+The usual symptom is error messages like this:
 `Error: encoding error in FILENAME 'ENCODING' codec can't decode byte ... in position ...: invalid start byte`
 
 Unfortunately, Python3 fails to provide useful built-ins to deal with this.
@@ -94,6 +94,40 @@ vulnerabilities in programs that cannot be built or cannot be linked.
 Flawfinder also doesn't get as confused by macro definitions
 and other oddities that more sophisticated tools have trouble with.
 
+# Flawfinder GitHub Action
+
+There's a GitHub action available for those who use GitHub.
+
+## Usage
+
+See [action.yml](https://github.com/david-a-wheeler/flawfinder/blob/main/action.yml)
+
+Create a .yml file under .github/workflows with the following contents:
+
+### Basic demo:
+
+```yml
+- name: flawfinder_scan
+  uses: david-a-wheeler/flawfinder@2.0.19
+  with:
+    arguments: '--sarif ./'
+    output: 'flawfinder_results.sarif'
+```
+
+You can add many other additions to the arguments.
+For example, `--error-level=4` will cause an error to be returned if
+flawfinder finds a vulnerability of level 4 or higher.
+Notice the version number after the `@` symbol; you can select a
+different version.
+
+You can find the action name and version string from [Marketplace](https://github.com/marketplace/actions/flawfinder_scan)
+by clicking "Use latest/xxx version" button.
+
+### Input options:
+
+- arguments: [Flawfinder command arguments](ttps://github.com/david-a-wheeler/flawfinder/blob/master/README.md#usage)
+- output: Flawfinder output file name. Can be uploaded to GitHub.
+
 # Contributions
 
 We love contributions!  For more information on contributing, see

action.yml

@@ -0,0 +1,17 @@
+name: 'flawfinder_scan'
+description: 'Execute Flawfinder to scan source code for vulnerabilities'
+inputs:
+  arguments:
+    description: 'Command arguments to be sent to Flawfinder'
+    required: true
+    default: ''
+  output:
+    description: 'Output file name'
+    required: true
+    default: ''
+runs:
+  using: 'docker'
+  image: 'Dockerfile'
+  args:
+    - ${{ inputs.arguments }}
+    - ${{ inputs.output }}

entrypoint.sh

@@ -0,0 +1,11 @@
+#!/bin/sh -l
+# $1 whitespace-separated arguments. Some filenames may need to be escaped.
+# $2 output filename
+
+output="${2:-flawfinder-output.txt}"
+
+flawfinder $1 > "$output"
+result="$?"
+
+cat "$output"
+exit "$result"

flawfinder.1

@@ -19,7 +19,7 @@
 .\"
 .\" Man page created 17 May 2001 by David A. Wheeler (dwheeler@dwheeler.com)
 .\"
-.TH FLAWFINDER 1 "4 Apr 2018" "Flawfinder" "Flawfinder"
+.TH FLAWFINDER 1 "3 Jun 2021" "Flawfinder" "Flawfinder"
 .SH NAME
 flawfinder \- lexically find potential security flaws ("hits") in source code
 .SH SYNOPSIS
@@ -52,6 +52,7 @@ flawfinder \- lexically find potential security flaws ("hits") in source code
 .RB [ \-\-dataonly | \-D ]
 .RB [ \-\-html | \-H ]
 .RB [ \-\-immediate | -i ]
+.RB [ \-\-sarif ]
 .RB [ \-\-singleline | \-S ]
 .RB [ \-\-omittime ]
 .RB [ \-\-quiet | \-Q ]
@@ -221,7 +222,10 @@ Not every hit is necessarily a security vulnerability, and
 there may be other security vulnerabilities not reported by the tool.
 .PP
 Flawfinder can easily integrate into a continuous integration system.
-You might want to check out the \-\-error\-level option to help do that.
+You might want to check out the \-\-error\-level option to help do that, e.g.,
+using \-\-error\-level=4 will cause an error to be returned if flawfinder
+finds a vulnerability of level 4 or higher.
+
 .PP
 Flawfinder is released under the GNU GPL license version 2 or later (GPLv2+).
 .PP
@@ -300,7 +304,8 @@ integrated development environments; see the examples for
 more information.
 .PP
 Flawfinder includes many other options, including ones to
-create HTML versions of the output (useful for prettier displays).
+create HTML versions of the output (useful for prettier displays) and
+OASIS Static Analysis Results Interchange Format (SARIF) output.
 The next section describes those options in more detail.
 
 .SH OPTIONS
@@ -534,6 +539,35 @@ Format the output as HTML instead of as simple text.
 .BI -i
 Immediately display hits (don't just wait until the end).
 
+
+.TP
+.BI \-\-sarif
+Produce output in the OASIS
+Static Analysis Results Interchange Format (SARIF) format (a JSON-based format).
+The goals of the SARIF format, as explained in
+version 2.1.0 (27 March 2020) of its specification, include being able to
+"comprehensively capture the range of data produced by commonly
+used static analysis tools."
+SARIF output identifies the tool name as "Flawfinder".
+The flawfinder levels 0 through 5 are mapped to SARIF rank (by dividing by 5),
+SARIF level, and the default viewer action as follows:
+
+Flawfinder 0: SARIF rank 0.0, SARIF level note, Does not display by default
+
+Flawfinder 1: SARIF rank 0.2, SARIF level note, Does not display by default
+
+Flawfinder 2: SARIF rank 0.4, SARIF level note, Does not display by default
+
+Flawfinder 3: SARIF rank 0.6, SARIF level warning, Displays by default, does not break build / other processes
+
+Flawfinder 4: SARIF rank 0.8, SARIF level error, Displays by default, breaks build/ other processes
+
+Flawfinder 5: SARIF rank 1.0, SARIF level error, Displays by default, breaks build/ other processes
+
+A big thanks to Yong Yan implementing SARIF output generation for flawfinder!
+For more about the SARIF format, see:
+https://www.oasis-open.org/committees/tc_home.php?wg_abbrev=sarif
+
 .TP
 .BI "\-\-singleline"
 .TP
@@ -702,6 +736,14 @@ reporting on all hits found.
 By default flawfinder will skip symbolic links and
 directories with names that start with a period.
 
+.TP
+.B "flawfinder \-\-error-level=4 ."
+Examine all the C/C++ files in the current directory
+and its subdirectories (recursively);
+return an error code if there are vulnerabilities
+level 4 and up (the two highest risk levels).
+This is a plausible way to use flawfinder in a continuous integration system.
+
 .TP
 .B "flawfinder \-\-minlevel=4 ."
 Examine all the C/C++ files in the current directory
@@ -1223,9 +1265,9 @@ This is not a bug, this is intended behavior.
 Flawfinder will report hits even if they are between
 a literal "#if 0" and "#endif".
 It would be possible to change this particular situation, but directly
-using "#if 0" to comment-out code (other than during debugging) is itself
-that the removal is very temporary (in which case we should report it) or
-an indicator of a problem with poor code practices.
+using "#if 0" to comment-out code (other than during debugging) indicates
+(1) the removal is very temporary (in which case we should still report it) or
+(2) very poor code practices.
 If you want to permanently get rid of code, then
 delete it instead of using "#if 0", since you can always see what it was
 using your version control software.

flawfinder.spec

@@ -1,6 +1,6 @@
 Name: flawfinder
 Summary: Examines C/C++ source code for security flaws
-Version: 2.0.15
+Version: 2.0.19
 Release: 1%{?dist}
 License: GPLv2+
 Group: Development/Tools

makefile

@@ -6,7 +6,7 @@
 # how to change version numbers.
 
 NAME=flawfinder
-VERSION=2.0.15
+VERSION=2.0.19
 RPM_VERSION=1
 VERSIONEDNAME=$(NAME)-$(VERSION)
 ARCH=noarch
@@ -66,14 +66,15 @@ INSTALL_DATA=cp -p
 # (admittedly rare) problem of bad date/timestamps causing the
 # compiled code to override later uncompiled Python code.
 install:
-	-$(MKDIR_P) $(DESTDIR)$(INSTALL_DIR_BIN)
-	$(INSTALL_PROGRAM) flawfinder $(DESTDIR)$(INSTALL_DIR_BIN)/flawfinder$(PYTHONEXT)
-	-$(MKDIR_P) $(DESTDIR)$(INSTALL_DIR_MAN)
-	$(INSTALL_DATA) flawfinder.1 $(DESTDIR)$(INSTALL_DIR_MAN)/flawfinder.1
+	-$(MKDIR_P) "$(DESTDIR)$(INSTALL_DIR_BIN)"
+	$(INSTALL_PROGRAM) flawfinder.py "$(DESTDIR)$(INSTALL_DIR_BIN)/flawfinder$(PYTHONEXT)"
+	chmod a+x "$(DESTDIR)$(INSTALL_DIR_BIN)/flawfinder$(PYTHONEXT)"
+	-$(MKDIR_P) "$(DESTDIR)$(INSTALL_DIR_MAN)"
+	$(INSTALL_DATA) flawfinder.1 "$(DESTDIR)$(INSTALL_DIR_MAN)/flawfinder.1"
 
 uninstall:
-	rm -f $(DESTDIR)$(INSTALL_DIR_BIN)/flawfinder$(PYTHONEXT)
-	rm -f $(DESTDIR)$(INSTALL_DIR_MAN)/flawfinder.1
+	rm -f "$(DESTDIR)$(INSTALL_DIR_BIN)/flawfinder$(PYTHONEXT)"
+	rm -f "$(DESTDIR)$(INSTALL_DIR_MAN)/flawfinder.1"
 
 flawfinder.1.gz: flawfinder.1
 	gzip -c9 < flawfinder.1 > flawfinder.1.gz
@@ -103,8 +104,6 @@ distribute: clean flawfinder.pdf flawfinder.ps
 	cp -r -p [a-zA-Z]* ,tempdir
 	rm -f ,tempdir/*.tar.gz
 	rm -f ,tempdir/*.rpm
-	# We don't need both "flawfinder" and "flawfinder.py":
-	rm -f ,tempdir/flawfinder.py
 	mv ,tempdir flawfinder-$(VERSION)
 	# Nobody else needs "update" either.
 	rm -f ,tempdir/update
@@ -129,7 +128,7 @@ upload-pypi:
 
 time:
 	echo "Timing the program. First, time taken:"
-	time ./flawfinder $(SAMPLE_DIR)/*/*.[ch] > /dev/null
+	time ./flawfinder.py $(SAMPLE_DIR)/*/*.[ch] > /dev/null
 	echo "Lines examined:"
 	wc -l $(SAMPLE_DIR)/*/*.[ch] | tail -2
 
@@ -145,7 +144,7 @@ test-is-correct:
 	cd $(TESTDIR); $(MAKE) test-is-correct
 
 profile:
-	/usr/lib/python1.5/profile.py ./flawfinder > profile-results $(SAMPLE_DIR)/*/*.[ch] > profile-results 
+	/usr/lib/python1.5/profile.py ./flawfinder.py > profile-results $(SAMPLE_DIR)/*/*.[ch] > profile-results
 
 
 rpm: distribute
@@ -184,7 +183,7 @@ cwe: cwe.c
 	$(CC) -o cwe cwe.c -lfl
 
 show-cwes: cwe
-	./cwe < flawfinder | sort -u -V
+	./cwe < flawfinder.py | sort -u -V
 
 pylint:
 	pylint flawfinder
@@ -196,4 +195,3 @@ pylint:
 # file into a subdirectory (named flawfinder-versionnumber).
 # I can then create all the distribution files by just typing:
 #  python setup.py bdist_rpm
-

release_process.md

@@ -9,7 +9,7 @@ Make sure every release has a unique version number.
 
 To change version number, edit the following files:
 makefile
-flawfinder
+flawfinder.py
 flawfinder.spec
 setup.py
 index.html # in dwheeler.com/flawfinder

setup.py

@@ -9,7 +9,7 @@ from setuptools import setup # Don't need find_packages
 
 setup (# Distribution meta-data
        name = "flawfinder",
-       version = "2.0.15",
+       version = "2.0.19",
        # We install a script, not a separate package.
        # packages = ["flawfinder"], # Must be same as name
        # Do not need: packages=find_packages(),
@@ -41,7 +41,11 @@ It is released under the GNU GPL license.""",
            'Topic :: Software Development :: Testing'
            ],
        python_requires = '>=2.7',
-       scripts = [ 'flawfinder' ],
+       entry_points={
+        'console_scripts': [
+            'flawfinder = flawfinder:main',
+        ],
+       },
        data_files = [ ('share/man/man1', [ 'flawfinder.1.gz' ]) ],
-       py_modules = [ ],
+       py_modules = ['flawfinder'],
       )

simplejson/__init__.py

@@ -0,0 +1,577 @@
+r"""JSON (JavaScript Object Notation) <http://json.org> is a subset of
+JavaScript syntax (ECMA-262 3rd edition) used as a lightweight data
+interchange format.
+
+:mod:`simplejson` exposes an API familiar to users of the standard library
+:mod:`marshal` and :mod:`pickle` modules. It is the externally maintained
+version of the :mod:`json` library contained in Python 2.6, but maintains
+compatibility back to Python 2.5 and (currently) has significant performance
+advantages, even without using the optional C extension for speedups.
+
+Encoding basic Python object hierarchies::
+
+    >>> import simplejson as json
+    >>> json.dumps(['foo', {'bar': ('baz', None, 1.0, 2)}])
+    '["foo", {"bar": ["baz", null, 1.0, 2]}]'
+    >>> print(json.dumps("\"foo\bar"))
+    "\"foo\bar"
+    >>> print(json.dumps(u'\u1234'))
+    "\u1234"
+    >>> print(json.dumps('\\'))
+    "\\"
+    >>> print(json.dumps({"c": 0, "b": 0, "a": 0}, sort_keys=True))
+    {"a": 0, "b": 0, "c": 0}
+    >>> from simplejson.compat import StringIO
+    >>> io = StringIO()
+    >>> json.dump(['streaming API'], io)
+    >>> io.getvalue()
+    '["streaming API"]'
+
+Compact encoding::
+
+    >>> import simplejson as json
+    >>> obj = [1,2,3,{'4': 5, '6': 7}]
+    >>> json.dumps(obj, separators=(',',':'), sort_keys=True)
+    '[1,2,3,{"4":5,"6":7}]'
+
+Pretty printing::
+
+    >>> import simplejson as json
+    >>> print(json.dumps({'4': 5, '6': 7}, sort_keys=True, indent='    '))
+    {
+        "4": 5,
+        "6": 7
+    }
+
+Decoding JSON::
+
+    >>> import simplejson as json
+    >>> obj = [u'foo', {u'bar': [u'baz', None, 1.0, 2]}]
+    >>> json.loads('["foo", {"bar":["baz", null, 1.0, 2]}]') == obj
+    True
+    >>> json.loads('"\\"foo\\bar"') == u'"foo\x08ar'
+    True
+    >>> from simplejson.compat import StringIO
+    >>> io = StringIO('["streaming API"]')
+    >>> json.load(io)[0] == 'streaming API'
+    True
+
+Specializing JSON object decoding::
+
+    >>> import simplejson as json
+    >>> def as_complex(dct):
+    ...     if '__complex__' in dct:
+    ...         return complex(dct['real'], dct['imag'])
+    ...     return dct
+    ...
+    >>> json.loads('{"__complex__": true, "real": 1, "imag": 2}',
+    ...     object_hook=as_complex)
+    (1+2j)
+    >>> from decimal import Decimal
+    >>> json.loads('1.1', parse_float=Decimal) == Decimal('1.1')
+    True
+
+Specializing JSON object encoding::
+
+    >>> import simplejson as json
+    >>> def encode_complex(obj):
+    ...     if isinstance(obj, complex):
+    ...         return [obj.real, obj.imag]
+    ...     raise TypeError('Object of type %s is not JSON serializable' %
+    ...                     obj.__class__.__name__)
+    ...
+    >>> json.dumps(2 + 1j, default=encode_complex)
+    '[2.0, 1.0]'
+    >>> json.JSONEncoder(default=encode_complex).encode(2 + 1j)
+    '[2.0, 1.0]'
+    >>> ''.join(json.JSONEncoder(default=encode_complex).iterencode(2 + 1j))
+    '[2.0, 1.0]'
+
+
+Using simplejson.tool from the shell to validate and pretty-print::
+
+    $ echo '{"json":"obj"}' | python -m simplejson.tool
+    {
+        "json": "obj"
+    }
+    $ echo '{ 1.2:3.4}' | python -m simplejson.tool
+    Expecting property name: line 1 column 3 (char 2)
+"""
+from __future__ import absolute_import
+__version__ = '3.16.0'
+__all__ = [
+    'dump', 'dumps', 'load', 'loads',
+    'JSONDecoder', 'JSONDecodeError', 'JSONEncoder',
+    'OrderedDict', 'simple_first', 'RawJSON'
+]
+
+__author__ = 'Bob Ippolito <bob@redivi.com>'
+
+from decimal import Decimal
+
+from .errors import JSONDecodeError
+from .raw_json import RawJSON
+from .decoder import JSONDecoder
+from .encoder import JSONEncoder, JSONEncoderForHTML
+def _import_OrderedDict():
+    import collections
+    try:
+        return collections.OrderedDict
+    except AttributeError:
+        from . import ordered_dict
+        return ordered_dict.OrderedDict
+OrderedDict = _import_OrderedDict()
+
+def _import_c_make_encoder():
+    try:
+        from ._speedups import make_encoder
+        return make_encoder
+    except ImportError:
+        return None
+
+_default_encoder = JSONEncoder(
+    skipkeys=False,
+    ensure_ascii=True,
+    check_circular=True,
+    allow_nan=True,
+    indent=None,
+    separators=None,
+    encoding='utf-8',
+    default=None,
+    use_decimal=True,
+    namedtuple_as_object=True,
+    tuple_as_array=True,
+    iterable_as_array=False,
+    bigint_as_string=False,
+    item_sort_key=None,
+    for_json=False,
+    ignore_nan=False,
+    int_as_string_bitcount=None,
+)
+
+def dump(obj, fp, skipkeys=False, ensure_ascii=True, check_circular=True,
+         allow_nan=True, cls=None, indent=None, separators=None,
+         encoding='utf-8', default=None, use_decimal=True,
+         namedtuple_as_object=True, tuple_as_array=True,
+         bigint_as_string=False, sort_keys=False, item_sort_key=None,
+         for_json=False, ignore_nan=False, int_as_string_bitcount=None,
+         iterable_as_array=False, **kw):
+    """Serialize ``obj`` as a JSON formatted stream to ``fp`` (a
+    ``.write()``-supporting file-like object).
+
+    If *skipkeys* is true then ``dict`` keys that are not basic types
+    (``str``, ``unicode``, ``int``, ``long``, ``float``, ``bool``, ``None``)
+    will be skipped instead of raising a ``TypeError``.
+
+    If *ensure_ascii* is false, then the some chunks written to ``fp``
+    may be ``unicode`` instances, subject to normal Python ``str`` to
+    ``unicode`` coercion rules. Unless ``fp.write()`` explicitly
+    understands ``unicode`` (as in ``codecs.getwriter()``) this is likely
+    to cause an error.
+
+    If *check_circular* is false, then the circular reference check
+    for container types will be skipped and a circular reference will
+    result in an ``OverflowError`` (or worse).
+
+    If *allow_nan* is false, then it will be a ``ValueError`` to
+    serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``)
+    in strict compliance of the original JSON specification, instead of using
+    the JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``). See
+    *ignore_nan* for ECMA-262 compliant behavior.
+
+    If *indent* is a string, then JSON array elements and object members
+    will be pretty-printed with a newline followed by that string repeated
+    for each level of nesting. ``None`` (the default) selects the most compact
+    representation without any newlines. For backwards compatibility with
+    versions of simplejson earlier than 2.1.0, an integer is also accepted
+    and is converted to a string with that many spaces.
+
+    If specified, *separators* should be an
+    ``(item_separator, key_separator)`` tuple.  The default is ``(', ', ': ')``
+    if *indent* is ``None`` and ``(',', ': ')`` otherwise.  To get the most
+    compact JSON representation, you should specify ``(',', ':')`` to eliminate
+    whitespace.
+
+    *encoding* is the character encoding for str instances, default is UTF-8.
+
+    *default(obj)* is a function that should return a serializable version
+    of obj or raise ``TypeError``. The default simply raises ``TypeError``.
+
+    If *use_decimal* is true (default: ``True``) then decimal.Decimal
+    will be natively serialized to JSON with full precision.
+
+    If *namedtuple_as_object* is true (default: ``True``),
+    :class:`tuple` subclasses with ``_asdict()`` methods will be encoded
+    as JSON objects.
+
+    If *tuple_as_array* is true (default: ``True``),
+    :class:`tuple` (and subclasses) will be encoded as JSON arrays.
+
+    If *iterable_as_array* is true (default: ``False``),
+    any object not in the above table that implements ``__iter__()``
+    will be encoded as a JSON array.
+
+    If *bigint_as_string* is true (default: ``False``), ints 2**53 and higher
+    or lower than -2**53 will be encoded as strings. This is to avoid the
+    rounding that happens in Javascript otherwise. Note that this is still a
+    lossy operation that will not round-trip correctly and should be used
+    sparingly.
+
+    If *int_as_string_bitcount* is a positive number (n), then int of size
+    greater than or equal to 2**n or lower than or equal to -2**n will be
+    encoded as strings.
+
+    If specified, *item_sort_key* is a callable used to sort the items in
+    each dictionary. This is useful if you want to sort items other than
+    in alphabetical order by key. This option takes precedence over
+    *sort_keys*.
+
+    If *sort_keys* is true (default: ``False``), the output of dictionaries
+    will be sorted by item.
+
+    If *for_json* is true (default: ``False``), objects with a ``for_json()``
+    method will use the return value of that method for encoding as JSON
+    instead of the object.
+
+    If *ignore_nan* is true (default: ``False``), then out of range
+    :class:`float` values (``nan``, ``inf``, ``-inf``) will be serialized as
+    ``null`` in compliance with the ECMA-262 specification. If true, this will
+    override *allow_nan*.
+
+    To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the
+    ``.default()`` method to serialize additional types), specify it with
+    the ``cls`` kwarg. NOTE: You should use *default* or *for_json* instead
+    of subclassing whenever possible.
+
+    """
+    # cached encoder
+    if (not skipkeys and ensure_ascii and
+        check_circular and allow_nan and
+        cls is None and indent is None and separators is None and
+        encoding == 'utf-8' and default is None and use_decimal
+        and namedtuple_as_object and tuple_as_array and not iterable_as_array
+        and not bigint_as_string and not sort_keys
+        and not item_sort_key and not for_json
+        and not ignore_nan and int_as_string_bitcount is None
+        and not kw
+    ):
+        iterable = _default_encoder.iterencode(obj)
+    else:
+        if cls is None:
+            cls = JSONEncoder
+        iterable = cls(skipkeys=skipkeys, ensure_ascii=ensure_ascii,
+            check_circular=check_circular, allow_nan=allow_nan, indent=indent,
+            separators=separators, encoding=encoding,
+            default=default, use_decimal=use_decimal,
+            namedtuple_as_object=namedtuple_as_object,
+            tuple_as_array=tuple_as_array,
+            iterable_as_array=iterable_as_array,
+            bigint_as_string=bigint_as_string,
+            sort_keys=sort_keys,
+            item_sort_key=item_sort_key,
+            for_json=for_json,
+            ignore_nan=ignore_nan,
+            int_as_string_bitcount=int_as_string_bitcount,
+            **kw).iterencode(obj)
+    # could accelerate with writelines in some versions of Python, at
+    # a debuggability cost
+    for chunk in iterable:
+        fp.write(chunk)
+
+
+def dumps(obj, skipkeys=False, ensure_ascii=True, check_circular=True,
+          allow_nan=True, cls=None, indent=None, separators=None,
+          encoding='utf-8', default=None, use_decimal=True,
+          namedtuple_as_object=True, tuple_as_array=True,
+          bigint_as_string=False, sort_keys=False, item_sort_key=None,
+          for_json=False, ignore_nan=False, int_as_string_bitcount=None,
+          iterable_as_array=False, **kw):
+    """Serialize ``obj`` to a JSON formatted ``str``.
+
+    If ``skipkeys`` is false then ``dict`` keys that are not basic types
+    (``str``, ``unicode``, ``int``, ``long``, ``float``, ``bool``, ``None``)
+    will be skipped instead of raising a ``TypeError``.
+
+    If ``ensure_ascii`` is false, then the return value will be a
+    ``unicode`` instance subject to normal Python ``str`` to ``unicode``
+    coercion rules instead of being escaped to an ASCII ``str``.
+
+    If ``check_circular`` is false, then the circular reference check
+    for container types will be skipped and a circular reference will
+    result in an ``OverflowError`` (or worse).
+
+    If ``allow_nan`` is false, then it will be a ``ValueError`` to
+    serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``) in
+    strict compliance of the JSON specification, instead of using the
+    JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``).
+
+    If ``indent`` is a string, then JSON array elements and object members
+    will be pretty-printed with a newline followed by that string repeated
+    for each level of nesting. ``None`` (the default) selects the most compact
+    representation without any newlines. For backwards compatibility with
+    versions of simplejson earlier than 2.1.0, an integer is also accepted
+    and is converted to a string with that many spaces.
+
+    If specified, ``separators`` should be an
+    ``(item_separator, key_separator)`` tuple.  The default is ``(', ', ': ')``
+    if *indent* is ``None`` and ``(',', ': ')`` otherwise.  To get the most
+    compact JSON representation, you should specify ``(',', ':')`` to eliminate
+    whitespace.
+
+    ``encoding`` is the character encoding for str instances, default is UTF-8.
+
+    ``default(obj)`` is a function that should return a serializable version
+    of obj or raise TypeError. The default simply raises TypeError.
+
+    If *use_decimal* is true (default: ``True``) then decimal.Decimal
+    will be natively serialized to JSON with full precision.
+
+    If *namedtuple_as_object* is true (default: ``True``),
+    :class:`tuple` subclasses with ``_asdict()`` methods will be encoded
+    as JSON objects.
+
+    If *tuple_as_array* is true (default: ``True``),
+    :class:`tuple` (and subclasses) will be encoded as JSON arrays.
+
+    If *iterable_as_array* is true (default: ``False``),
+    any object not in the above table that implements ``__iter__()``
+    will be encoded as a JSON array.
+
+    If *bigint_as_string* is true (not the default), ints 2**53 and higher
+    or lower than -2**53 will be encoded as strings. This is to avoid the
+    rounding that happens in Javascript otherwise.
+
+    If *int_as_string_bitcount* is a positive number (n), then int of size
+    greater than or equal to 2**n or lower than or equal to -2**n will be
+    encoded as strings.
+
+    If specified, *item_sort_key* is a callable used to sort the items in
+    each dictionary. This is useful if you want to sort items other than
+    in alphabetical order by key. This option takes precendence over
+    *sort_keys*.
+
+    If *sort_keys* is true (default: ``False``), the output of dictionaries
+    will be sorted by item.
+
+    If *for_json* is true (default: ``False``), objects with a ``for_json()``
+    method will use the return value of that method for encoding as JSON
+    instead of the object.
+
+    If *ignore_nan* is true (default: ``False``), then out of range
+    :class:`float` values (``nan``, ``inf``, ``-inf``) will be serialized as
+    ``null`` in compliance with the ECMA-262 specification. If true, this will
+    override *allow_nan*.
+
+    To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the
+    ``.default()`` method to serialize additional types), specify it with
+    the ``cls`` kwarg. NOTE: You should use *default* instead of subclassing
+    whenever possible.
+
+    """
+    # cached encoder
+    if (not skipkeys and ensure_ascii and
+        check_circular and allow_nan and
+        cls is None and indent is None and separators is None and
+        encoding == 'utf-8' and default is None and use_decimal
+        and namedtuple_as_object and tuple_as_array and not iterable_as_array
+        and not bigint_as_string and not sort_keys
+        and not item_sort_key and not for_json
+        and not ignore_nan and int_as_string_bitcount is None
+        and not kw
+    ):
+        return _default_encoder.encode(obj)
+    if cls is None:
+        cls = JSONEncoder
+    return cls(
+        skipkeys=skipkeys, ensure_ascii=ensure_ascii,
+        check_circular=check_circular, allow_nan=allow_nan, indent=indent,
+        separators=separators, encoding=encoding, default=default,
+        use_decimal=use_decimal,
+        namedtuple_as_object=namedtuple_as_object,
+        tuple_as_array=tuple_as_array,
+        iterable_as_array=iterable_as_array,
+        bigint_as_string=bigint_as_string,
+        sort_keys=sort_keys,
+        item_sort_key=item_sort_key,
+        for_json=for_json,
+        ignore_nan=ignore_nan,
+        int_as_string_bitcount=int_as_string_bitcount,
+        **kw).encode(obj)
+
+
+_default_decoder = JSONDecoder(encoding=None, object_hook=None,
+                               object_pairs_hook=None)
+
+
+def load(fp, encoding=None, cls=None, object_hook=None, parse_float=None,
+        parse_int=None, parse_constant=None, object_pairs_hook=None,
+        use_decimal=False, namedtuple_as_object=True, tuple_as_array=True,
+        **kw):
+    """Deserialize ``fp`` (a ``.read()``-supporting file-like object containing
+    a JSON document) to a Python object.
+
+    *encoding* determines the encoding used to interpret any
+    :class:`str` objects decoded by this instance (``'utf-8'`` by
+    default).  It has no effect when decoding :class:`unicode` objects.
+
+    Note that currently only encodings that are a superset of ASCII work,
+    strings of other encodings should be passed in as :class:`unicode`.
+
+    *object_hook*, if specified, will be called with the result of every
+    JSON object decoded and its return value will be used in place of the
+    given :class:`dict`.  This can be used to provide custom
+    deserializations (e.g. to support JSON-RPC class hinting).
+
+    *object_pairs_hook* is an optional function that will be called with
+    the result of any object literal decode with an ordered list of pairs.
+    The return value of *object_pairs_hook* will be used instead of the
+    :class:`dict`.  This feature can be used to implement custom decoders
+    that rely on the order that the key and value pairs are decoded (for
+    example, :func:`collections.OrderedDict` will remember the order of
+    insertion). If *object_hook* is also defined, the *object_pairs_hook*
+    takes priority.
+
+    *parse_float*, if specified, will be called with the string of every
+    JSON float to be decoded.  By default, this is equivalent to
+    ``float(num_str)``. This can be used to use another datatype or parser
+    for JSON floats (e.g. :class:`decimal.Decimal`).
+
+    *parse_int*, if specified, will be called with the string of every
+    JSON int to be decoded.  By default, this is equivalent to
+    ``int(num_str)``.  This can be used to use another datatype or parser
+    for JSON integers (e.g. :class:`float`).
+
+    *parse_constant*, if specified, will be called with one of the
+    following strings: ``'-Infinity'``, ``'Infinity'``, ``'NaN'``.  This
+    can be used to raise an exception if invalid JSON numbers are
+    encountered.
+
+    If *use_decimal* is true (default: ``False``) then it implies
+    parse_float=decimal.Decimal for parity with ``dump``.
+
+    To use a custom ``JSONDecoder`` subclass, specify it with the ``cls``
+    kwarg. NOTE: You should use *object_hook* or *object_pairs_hook* instead
+    of subclassing whenever possible.
+
+    """
+    return loads(fp.read(),
+        encoding=encoding, cls=cls, object_hook=object_hook,
+        parse_float=parse_float, parse_int=parse_int,
+        parse_constant=parse_constant, object_pairs_hook=object_pairs_hook,
+        use_decimal=use_decimal, **kw)
+
+
+def loads(s, encoding=None, cls=None, object_hook=None, parse_float=None,
+        parse_int=None, parse_constant=None, object_pairs_hook=None,
+        use_decimal=False, **kw):
+    """Deserialize ``s`` (a ``str`` or ``unicode`` instance containing a JSON
+    document) to a Python object.
+
+    *encoding* determines the encoding used to interpret any
+    :class:`str` objects decoded by this instance (``'utf-8'`` by
+    default).  It has no effect when decoding :class:`unicode` objects.
+
+    Note that currently only encodings that are a superset of ASCII work,
+    strings of other encodings should be passed in as :class:`unicode`.
+
+    *object_hook*, if specified, will be called with the result of every
+    JSON object decoded and its return value will be used in place of the
+    given :class:`dict`.  This can be used to provide custom
+    deserializations (e.g. to support JSON-RPC class hinting).
+
+    *object_pairs_hook* is an optional function that will be called with
+    the result of any object literal decode with an ordered list of pairs.
+    The return value of *object_pairs_hook* will be used instead of the
+    :class:`dict`.  This feature can be used to implement custom decoders
+    that rely on the order that the key and value pairs are decoded (for
+    example, :func:`collections.OrderedDict` will remember the order of
+    insertion). If *object_hook* is also defined, the *object_pairs_hook*
+    takes priority.
+
+    *parse_float*, if specified, will be called with the string of every
+    JSON float to be decoded.  By default, this is equivalent to
+    ``float(num_str)``. This can be used to use another datatype or parser
+    for JSON floats (e.g. :class:`decimal.Decimal`).
+
+    *parse_int*, if specified, will be called with the string of every
+    JSON int to be decoded.  By default, this is equivalent to
+    ``int(num_str)``.  This can be used to use another datatype or parser
+    for JSON integers (e.g. :class:`float`).
+
+    *parse_constant*, if specified, will be called with one of the
+    following strings: ``'-Infinity'``, ``'Infinity'``, ``'NaN'``.  This
+    can be used to raise an exception if invalid JSON numbers are
+    encountered.
+
+    If *use_decimal* is true (default: ``False``) then it implies
+    parse_float=decimal.Decimal for parity with ``dump``.
+
+    To use a custom ``JSONDecoder`` subclass, specify it with the ``cls``
+    kwarg. NOTE: You should use *object_hook* or *object_pairs_hook* instead
+    of subclassing whenever possible.
+
+    """
+    if (cls is None and encoding is None and object_hook is None and
+            parse_int is None and parse_float is None and
+            parse_constant is None and object_pairs_hook is None
+            and not use_decimal and not kw):
+        return _default_decoder.decode(s)
+    if cls is None:
+        cls = JSONDecoder
+    if object_hook is not None:
+        kw['object_hook'] = object_hook
+    if object_pairs_hook is not None:
+        kw['object_pairs_hook'] = object_pairs_hook
+    if parse_float is not None:
+        kw['parse_float'] = parse_float
+    if parse_int is not None:
+        kw['parse_int'] = parse_int
+    if parse_constant is not None:
+        kw['parse_constant'] = parse_constant
+    if use_decimal:
+        if parse_float is not None:
+            raise TypeError("use_decimal=True implies parse_float=Decimal")
+        kw['parse_float'] = Decimal
+    return cls(encoding=encoding, **kw).decode(s)
+
+
+def _toggle_speedups(enabled):
+    from . import decoder as dec
+    from . import encoder as enc
+    from . import scanner as scan
+    c_make_encoder = _import_c_make_encoder()
+    if enabled:
+        dec.scanstring = dec.c_scanstring or dec.py_scanstring
+        enc.c_make_encoder = c_make_encoder
+        enc.encode_basestring_ascii = (enc.c_encode_basestring_ascii or
+            enc.py_encode_basestring_ascii)
+        scan.make_scanner = scan.c_make_scanner or scan.py_make_scanner
+    else:
+        dec.scanstring = dec.py_scanstring
+        enc.c_make_encoder = None
+        enc.encode_basestring_ascii = enc.py_encode_basestring_ascii
+        scan.make_scanner = scan.py_make_scanner
+    dec.make_scanner = scan.make_scanner
+    global _default_decoder
+    _default_decoder = JSONDecoder(
+        encoding=None,
+        object_hook=None,
+        object_pairs_hook=None,
+    )
+    global _default_encoder
+    _default_encoder = JSONEncoder(
+       skipkeys=False,
+       ensure_ascii=True,
+       check_circular=True,
+       allow_nan=True,
+       indent=None,
+       separators=None,
+       encoding='utf-8',
+       default=None,
+   )
+
+def simple_first(kv):
+    """Helper function to pass to item_sort_key to sort simple
+    elements to the top, then container elements.
+    """
+    return (isinstance(kv[1], (list, dict, tuple)), kv[0])

simplejson/compat.py

@@ -0,0 +1,34 @@
+"""Python 3 compatibility shims
+"""
+import sys
+if sys.version_info[0] < 3:
+    PY3 = False
+    def b(s):
+        return s
+    try:
+        from cStringIO import StringIO
+    except ImportError:
+        from StringIO import StringIO
+    BytesIO = StringIO
+    text_type = unicode
+    binary_type = str
+    string_types = (basestring,)
+    integer_types = (int, long)
+    unichr = unichr
+    reload_module = reload
+else:
+    PY3 = True
+    if sys.version_info[:2] >= (3, 4):
+        from importlib import reload as reload_module
+    else:
+        from imp import reload as reload_module
+    def b(s):
+        return bytes(s, 'latin1')
+    from io import StringIO, BytesIO
+    text_type = str
+    binary_type = bytes
+    string_types = (str,)
+    integer_types = (int,)
+    unichr = chr
+
+long_type = integer_types[-1]

simplejson/decoder.py

@@ -0,0 +1,400 @@
+"""Implementation of JSONDecoder
+"""
+from __future__ import absolute_import
+import re
+import sys
+import struct
+from .compat import PY3, unichr
+from .scanner import make_scanner, JSONDecodeError
+
+def _import_c_scanstring():
+    try:
+        from ._speedups import scanstring
+        return scanstring
+    except ImportError:
+        return None
+c_scanstring = _import_c_scanstring()
+
+# NOTE (3.1.0): JSONDecodeError may still be imported from this module for
+# compatibility, but it was never in the __all__
+__all__ = ['JSONDecoder']
+
+FLAGS = re.VERBOSE | re.MULTILINE | re.DOTALL
+
+def _floatconstants():
+    if sys.version_info < (2, 6):
+        _BYTES = '7FF80000000000007FF0000000000000'.decode('hex')
+        nan, inf = struct.unpack('>dd', _BYTES)
+    else:
+        nan = float('nan')
+        inf = float('inf')
+    return nan, inf, -inf
+
+NaN, PosInf, NegInf = _floatconstants()
+
+_CONSTANTS = {
+    '-Infinity': NegInf,
+    'Infinity': PosInf,
+    'NaN': NaN,
+}
+
+STRINGCHUNK = re.compile(r'(.*?)(["\\\x00-\x1f])', FLAGS)
+BACKSLASH = {
+    '"': u'"', '\\': u'\\', '/': u'/',
+    'b': u'\b', 'f': u'\f', 'n': u'\n', 'r': u'\r', 't': u'\t',
+}
+
+DEFAULT_ENCODING = "utf-8"
+
+def py_scanstring(s, end, encoding=None, strict=True,
+        _b=BACKSLASH, _m=STRINGCHUNK.match, _join=u''.join,
+        _PY3=PY3, _maxunicode=sys.maxunicode):
+    """Scan the string s for a JSON string. End is the index of the
+    character in s after the quote that started the JSON string.
+    Unescapes all valid JSON string escape sequences and raises ValueError
+    on attempt to decode an invalid string. If strict is False then literal
+    control characters are allowed in the string.
+
+    Returns a tuple of the decoded string and the index of the character in s
+    after the end quote."""
+    if encoding is None:
+        encoding = DEFAULT_ENCODING
+    chunks = []
+    _append = chunks.append
+    begin = end - 1
+    while 1:
+        chunk = _m(s, end)
+        if chunk is None:
+            raise JSONDecodeError(
+                "Unterminated string starting at", s, begin)
+        end = chunk.end()
+        content, terminator = chunk.groups()
+        # Content is contains zero or more unescaped string characters
+        if content:
+            if not _PY3 and not isinstance(content, unicode):
+                content = unicode(content, encoding)
+            _append(content)
+        # Terminator is the end of string, a literal control character,
+        # or a backslash denoting that an escape sequence follows
+        if terminator == '"':
+            break
+        elif terminator != '\\':
+            if strict:
+                msg = "Invalid control character %r at"
+                raise JSONDecodeError(msg, s, end)
+            else:
+                _append(terminator)
+                continue
+        try:
+            esc = s[end]
+        except IndexError:
+            raise JSONDecodeError(
+                "Unterminated string starting at", s, begin)
+        # If not a unicode escape sequence, must be in the lookup table
+        if esc != 'u':
+            try:
+                char = _b[esc]
+            except KeyError:
+                msg = "Invalid \\X escape sequence %r"
+                raise JSONDecodeError(msg, s, end)
+            end += 1
+        else:
+            # Unicode escape sequence
+            msg = "Invalid \\uXXXX escape sequence"
+            esc = s[end + 1:end + 5]
+            escX = esc[1:2]
+            if len(esc) != 4 or escX == 'x' or escX == 'X':
+                raise JSONDecodeError(msg, s, end - 1)
+            try:
+                uni = int(esc, 16)
+            except ValueError:
+                raise JSONDecodeError(msg, s, end - 1)
+            end += 5
+            # Check for surrogate pair on UCS-4 systems
+            # Note that this will join high/low surrogate pairs
+            # but will also pass unpaired surrogates through
+            if (_maxunicode > 65535 and
+                uni & 0xfc00 == 0xd800 and
+                s[end:end + 2] == '\\u'):
+                esc2 = s[end + 2:end + 6]
+                escX = esc2[1:2]
+                if len(esc2) == 4 and not (escX == 'x' or escX == 'X'):
+                    try:
+                        uni2 = int(esc2, 16)
+                    except ValueError:
+                        raise JSONDecodeError(msg, s, end)
+                    if uni2 & 0xfc00 == 0xdc00:
+                        uni = 0x10000 + (((uni - 0xd800) << 10) |
+                                         (uni2 - 0xdc00))
+                        end += 6
+            char = unichr(uni)
+        # Append the unescaped character
+        _append(char)
+    return _join(chunks), end
+
+
+# Use speedup if available
+scanstring = c_scanstring or py_scanstring
+
+WHITESPACE = re.compile(r'[ \t\n\r]*', FLAGS)
+WHITESPACE_STR = ' \t\n\r'
+
+def JSONObject(state, encoding, strict, scan_once, object_hook,
+        object_pairs_hook, memo=None,
+        _w=WHITESPACE.match, _ws=WHITESPACE_STR):
+    (s, end) = state
+    # Backwards compatibility
+    if memo is None:
+        memo = {}
+    memo_get = memo.setdefault
+    pairs = []
+    # Use a slice to prevent IndexError from being raised, the following
+    # check will raise a more specific ValueError if the string is empty
+    nextchar = s[end:end + 1]
+    # Normally we expect nextchar == '"'
+    if nextchar != '"':
+        if nextchar in _ws:
+            end = _w(s, end).end()
+            nextchar = s[end:end + 1]
+        # Trivial empty object
+        if nextchar == '}':
+            if object_pairs_hook is not None:
+                result = object_pairs_hook(pairs)
+                return result, end + 1
+            pairs = {}
+            if object_hook is not None:
+                pairs = object_hook(pairs)
+            return pairs, end + 1
+        elif nextchar != '"':
+            raise JSONDecodeError(
+                "Expecting property name enclosed in double quotes",
+                s, end)
+    end += 1
+    while True:
+        key, end = scanstring(s, end, encoding, strict)
+        key = memo_get(key, key)
+
+        # To skip some function call overhead we optimize the fast paths where
+        # the JSON key separator is ": " or just ":".
+        if s[end:end + 1] != ':':
+            end = _w(s, end).end()
+            if s[end:end + 1] != ':':
+                raise JSONDecodeError("Expecting ':' delimiter", s, end)
+
+        end += 1
+
+        try:
+            if s[end] in _ws:
+                end += 1
+                if s[end] in _ws:
+                    end = _w(s, end + 1).end()
+        except IndexError:
+            pass
+
+        value, end = scan_once(s, end)
+        pairs.append((key, value))
+
+        try:
+            nextchar = s[end]
+            if nextchar in _ws:
+                end = _w(s, end + 1).end()
+                nextchar = s[end]
+        except IndexError:
+            nextchar = ''
+        end += 1
+
+        if nextchar == '}':
+            break
+        elif nextchar != ',':
+            raise JSONDecodeError("Expecting ',' delimiter or '}'", s, end - 1)
+
+        try:
+            nextchar = s[end]
+            if nextchar in _ws:
+                end += 1
+                nextchar = s[end]
+                if nextchar in _ws:
+                    end = _w(s, end + 1).end()
+                    nextchar = s[end]
+        except IndexError:
+            nextchar = ''
+
+        end += 1
+        if nextchar != '"':
+            raise JSONDecodeError(
+                "Expecting property name enclosed in double quotes",
+                s, end - 1)
+
+    if object_pairs_hook is not None:
+        result = object_pairs_hook(pairs)
+        return result, end
+    pairs = dict(pairs)
+    if object_hook is not None:
+        pairs = object_hook(pairs)
+    return pairs, end
+
+def JSONArray(state, scan_once, _w=WHITESPACE.match, _ws=WHITESPACE_STR):
+    (s, end) = state
+    values = []
+    nextchar = s[end:end + 1]
+    if nextchar in _ws:
+        end = _w(s, end + 1).end()
+        nextchar = s[end:end + 1]
+    # Look-ahead for trivial empty array
+    if nextchar == ']':
+        return values, end + 1
+    elif nextchar == '':
+        raise JSONDecodeError("Expecting value or ']'", s, end)
+    _append = values.append
+    while True:
+        value, end = scan_once(s, end)
+        _append(value)
+        nextchar = s[end:end + 1]
+        if nextchar in _ws:
+            end = _w(s, end + 1).end()
+            nextchar = s[end:end + 1]
+        end += 1
+        if nextchar == ']':
+            break
+        elif nextchar != ',':
+            raise JSONDecodeError("Expecting ',' delimiter or ']'", s, end - 1)
+
+        try:
+            if s[end] in _ws:
+                end += 1
+                if s[end] in _ws:
+                    end = _w(s, end + 1).end()
+        except IndexError:
+            pass
+
+    return values, end
+
+class JSONDecoder(object):
+    """Simple JSON <http://json.org> decoder
+
+    Performs the following translations in decoding by default:
+
+    +---------------+-------------------+
+    | JSON          | Python            |
+    +===============+===================+
+    | object        | dict              |
+    +---------------+-------------------+
+    | array         | list              |
+    +---------------+-------------------+
+    | string        | str, unicode      |
+    +---------------+-------------------+
+    | number (int)  | int, long         |
+    +---------------+-------------------+
+    | number (real) | float             |
+    +---------------+-------------------+
+    | true          | True              |
+    +---------------+-------------------+
+    | false         | False             |
+    +---------------+-------------------+
+    | null          | None              |
+    +---------------+-------------------+
+
+    It also understands ``NaN``, ``Infinity``, and ``-Infinity`` as
+    their corresponding ``float`` values, which is outside the JSON spec.
+
+    """
+
+    def __init__(self, encoding=None, object_hook=None, parse_float=None,
+            parse_int=None, parse_constant=None, strict=True,
+            object_pairs_hook=None):
+        """
+        *encoding* determines the encoding used to interpret any
+        :class:`str` objects decoded by this instance (``'utf-8'`` by
+        default).  It has no effect when decoding :class:`unicode` objects.
+
+        Note that currently only encodings that are a superset of ASCII work,
+        strings of other encodings should be passed in as :class:`unicode`.
+
+        *object_hook*, if specified, will be called with the result of every
+        JSON object decoded and its return value will be used in place of the
+        given :class:`dict`.  This can be used to provide custom
+        deserializations (e.g. to support JSON-RPC class hinting).
+
+        *object_pairs_hook* is an optional function that will be called with
+        the result of any object literal decode with an ordered list of pairs.
+        The return value of *object_pairs_hook* will be used instead of the
+        :class:`dict`.  This feature can be used to implement custom decoders
+        that rely on the order that the key and value pairs are decoded (for
+        example, :func:`collections.OrderedDict` will remember the order of
+        insertion). If *object_hook* is also defined, the *object_pairs_hook*
+        takes priority.
+
+        *parse_float*, if specified, will be called with the string of every
+        JSON float to be decoded.  By default, this is equivalent to
+        ``float(num_str)``. This can be used to use another datatype or parser
+        for JSON floats (e.g. :class:`decimal.Decimal`).
+
+        *parse_int*, if specified, will be called with the string of every
+        JSON int to be decoded.  By default, this is equivalent to
+        ``int(num_str)``.  This can be used to use another datatype or parser
+        for JSON integers (e.g. :class:`float`).
+
+        *parse_constant*, if specified, will be called with one of the
+        following strings: ``'-Infinity'``, ``'Infinity'``, ``'NaN'``.  This
+        can be used to raise an exception if invalid JSON numbers are
+        encountered.
+
+        *strict* controls the parser's behavior when it encounters an
+        invalid control character in a string. The default setting of
+        ``True`` means that unescaped control characters are parse errors, if
+        ``False`` then control characters will be allowed in strings.
+
+        """
+        if encoding is None:
+            encoding = DEFAULT_ENCODING
+        self.encoding = encoding
+        self.object_hook = object_hook
+        self.object_pairs_hook = object_pairs_hook
+        self.parse_float = parse_float or float
+        self.parse_int = parse_int or int
+        self.parse_constant = parse_constant or _CONSTANTS.__getitem__
+        self.strict = strict
+        self.parse_object = JSONObject
+        self.parse_array = JSONArray
+        self.parse_string = scanstring
+        self.memo = {}
+        self.scan_once = make_scanner(self)
+
+    def decode(self, s, _w=WHITESPACE.match, _PY3=PY3):
+        """Return the Python representation of ``s`` (a ``str`` or ``unicode``
+        instance containing a JSON document)
+
+        """
+        if _PY3 and isinstance(s, bytes):
+            s = str(s, self.encoding)
+        obj, end = self.raw_decode(s)
+        end = _w(s, end).end()
+        if end != len(s):
+            raise JSONDecodeError("Extra data", s, end, len(s))
+        return obj
+
+    def raw_decode(self, s, idx=0, _w=WHITESPACE.match, _PY3=PY3):
+        """Decode a JSON document from ``s`` (a ``str`` or ``unicode``
+        beginning with a JSON document) and return a 2-tuple of the Python
+        representation and the index in ``s`` where the document ended.
+        Optionally, ``idx`` can be used to specify an offset in ``s`` where
+        the JSON document begins.
+
+        This can be used to decode a JSON document from a string that may
+        have extraneous data at the end.
+
+        """
+        if idx < 0:
+            # Ensure that raw_decode bails on negative indexes, the regex
+            # would otherwise mask this behavior. #98
+            raise JSONDecodeError('Expecting value', s, idx)
+        if _PY3 and not isinstance(s, str):
+            raise TypeError("Input string must be text, not bytes")
+        # strip UTF-8 bom
+        if len(s) > idx:
+            ord0 = ord(s[idx])
+            if ord0 == 0xfeff:
+                idx += 1
+            elif ord0 == 0xef and s[idx:idx + 3] == '\xef\xbb\xbf':
+                idx += 3
+        return self.scan_once(s, idx=_w(s, idx).end())

simplejson/encoder.py

@@ -0,0 +1,722 @@
+"""Implementation of JSONEncoder
+"""
+from __future__ import absolute_import
+import re
+from operator import itemgetter
+# Do not import Decimal directly to avoid reload issues
+import decimal
+from .compat import unichr, binary_type, text_type, string_types, integer_types, PY3
+def _import_speedups():
+    try:
+        from . import _speedups
+        return _speedups.encode_basestring_ascii, _speedups.make_encoder
+    except ImportError:
+        return None, None
+c_encode_basestring_ascii, c_make_encoder = _import_speedups()
+
+from .decoder import PosInf
+from .raw_json import RawJSON
+
+ESCAPE = re.compile(r'[\x00-\x1f\\"]')
+ESCAPE_ASCII = re.compile(r'([\\"]|[^\ -~])')
+HAS_UTF8 = re.compile(r'[\x80-\xff]')
+ESCAPE_DCT = {
+    '\\': '\\\\',
+    '"': '\\"',
+    '\b': '\\b',
+    '\f': '\\f',
+    '\n': '\\n',
+    '\r': '\\r',
+    '\t': '\\t',
+}
+for i in range(0x20):
+    #ESCAPE_DCT.setdefault(chr(i), '\\u{0:04x}'.format(i))
+    ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,))
+
+FLOAT_REPR = repr
+
+def encode_basestring(s, _PY3=PY3, _q=u'"'):
+    """Return a JSON representation of a Python string
+
+    """
+    if _PY3:
+        if isinstance(s, bytes):
+            s = str(s, 'utf-8')
+        elif type(s) is not str:
+            # convert an str subclass instance to exact str
+            # raise a TypeError otherwise
+            s = str.__str__(s)
+    else:
+        if isinstance(s, str) and HAS_UTF8.search(s) is not None:
+            s = unicode(s, 'utf-8')
+        elif type(s) not in (str, unicode):
+            # convert an str subclass instance to exact str
+            # convert a unicode subclass instance to exact unicode
+            # raise a TypeError otherwise
+            if isinstance(s, str):
+                s = str.__str__(s)
+            else:
+                s = unicode.__getnewargs__(s)[0]
+    def replace(match):
+        return ESCAPE_DCT[match.group(0)]
+    return _q + ESCAPE.sub(replace, s) + _q
+
+
+def py_encode_basestring_ascii(s, _PY3=PY3):
+    """Return an ASCII-only JSON representation of a Python string
+
+    """
+    if _PY3:
+        if isinstance(s, bytes):
+            s = str(s, 'utf-8')
+        elif type(s) is not str:
+            # convert an str subclass instance to exact str
+            # raise a TypeError otherwise
+            s = str.__str__(s)
+    else:
+        if isinstance(s, str) and HAS_UTF8.search(s) is not None:
+            s = unicode(s, 'utf-8')
+        elif type(s) not in (str, unicode):
+            # convert an str subclass instance to exact str
+            # convert a unicode subclass instance to exact unicode
+            # raise a TypeError otherwise
+            if isinstance(s, str):
+                s = str.__str__(s)
+            else:
+                s = unicode.__getnewargs__(s)[0]
+    def replace(match):
+        s = match.group(0)
+        try:
+            return ESCAPE_DCT[s]
+        except KeyError:
+            n = ord(s)
+            if n < 0x10000:
+                #return '\\u{0:04x}'.format(n)
+                return '\\u%04x' % (n,)
+            else:
+                # surrogate pair
+                n -= 0x10000
+                s1 = 0xd800 | ((n >> 10) & 0x3ff)
+                s2 = 0xdc00 | (n & 0x3ff)
+                #return '\\u{0:04x}\\u{1:04x}'.format(s1, s2)
+                return '\\u%04x\\u%04x' % (s1, s2)
+    return '"' + str(ESCAPE_ASCII.sub(replace, s)) + '"'
+
+
+encode_basestring_ascii = (
+    c_encode_basestring_ascii or py_encode_basestring_ascii)
+
+class JSONEncoder(object):
+    """Extensible JSON <http://json.org> encoder for Python data structures.
+
+    Supports the following objects and types by default:
+
+    +-------------------+---------------+
+    | Python            | JSON          |
+    +===================+===============+
+    | dict, namedtuple  | object        |
+    +-------------------+---------------+
+    | list, tuple       | array         |
+    +-------------------+---------------+
+    | str, unicode      | string        |
+    +-------------------+---------------+
+    | int, long, float  | number        |
+    +-------------------+---------------+
+    | True              | true          |
+    +-------------------+---------------+
+    | False             | false         |
+    +-------------------+---------------+
+    | None              | null          |
+    +-------------------+---------------+
+
+    To extend this to recognize other objects, subclass and implement a
+    ``.default()`` method with another method that returns a serializable
+    object for ``o`` if possible, otherwise it should call the superclass
+    implementation (to raise ``TypeError``).
+
+    """
+    item_separator = ', '
+    key_separator = ': '
+
+    def __init__(self, skipkeys=False, ensure_ascii=True,
+                 check_circular=True, allow_nan=True, sort_keys=False,
+                 indent=None, separators=None, encoding='utf-8', default=None,
+                 use_decimal=True, namedtuple_as_object=True,
+                 tuple_as_array=True, bigint_as_string=False,
+                 item_sort_key=None, for_json=False, ignore_nan=False,
+                 int_as_string_bitcount=None, iterable_as_array=False):
+        """Constructor for JSONEncoder, with sensible defaults.
+
+        If skipkeys is false, then it is a TypeError to attempt
+        encoding of keys that are not str, int, long, float or None.  If
+        skipkeys is True, such items are simply skipped.
+
+        If ensure_ascii is true, the output is guaranteed to be str
+        objects with all incoming unicode characters escaped.  If
+        ensure_ascii is false, the output will be unicode object.
+
+        If check_circular is true, then lists, dicts, and custom encoded
+        objects will be checked for circular references during encoding to
+        prevent an infinite recursion (which would cause an OverflowError).
+        Otherwise, no such check takes place.
+
+        If allow_nan is true, then NaN, Infinity, and -Infinity will be
+        encoded as such.  This behavior is not JSON specification compliant,
+        but is consistent with most JavaScript based encoders and decoders.
+        Otherwise, it will be a ValueError to encode such floats.
+
+        If sort_keys is true, then the output of dictionaries will be
+        sorted by key; this is useful for regression tests to ensure
+        that JSON serializations can be compared on a day-to-day basis.
+
+        If indent is a string, then JSON array elements and object members
+        will be pretty-printed with a newline followed by that string repeated
+        for each level of nesting. ``None`` (the default) selects the most compact
+        representation without any newlines. For backwards compatibility with
+        versions of simplejson earlier than 2.1.0, an integer is also accepted
+        and is converted to a string with that many spaces.
+
+        If specified, separators should be an (item_separator, key_separator)
+        tuple.  The default is (', ', ': ') if *indent* is ``None`` and
+        (',', ': ') otherwise.  To get the most compact JSON representation,
+        you should specify (',', ':') to eliminate whitespace.
+
+        If specified, default is a function that gets called for objects
+        that can't otherwise be serialized.  It should return a JSON encodable
+        version of the object or raise a ``TypeError``.
+
+        If encoding is not None, then all input strings will be
+        transformed into unicode using that encoding prior to JSON-encoding.
+        The default is UTF-8.
+
+        If use_decimal is true (default: ``True``), ``decimal.Decimal`` will
+        be supported directly by the encoder. For the inverse, decode JSON
+        with ``parse_float=decimal.Decimal``.
+
+        If namedtuple_as_object is true (the default), objects with
+        ``_asdict()`` methods will be encoded as JSON objects.
+
+        If tuple_as_array is true (the default), tuple (and subclasses) will
+        be encoded as JSON arrays.
+
+        If *iterable_as_array* is true (default: ``False``),
+        any object not in the above table that implements ``__iter__()``
+        will be encoded as a JSON array.
+
+        If bigint_as_string is true (not the default), ints 2**53 and higher
+        or lower than -2**53 will be encoded as strings. This is to avoid the
+        rounding that happens in Javascript otherwise.
+
+        If int_as_string_bitcount is a positive number (n), then int of size
+        greater than or equal to 2**n or lower than or equal to -2**n will be
+        encoded as strings.
+
+        If specified, item_sort_key is a callable used to sort the items in
+        each dictionary. This is useful if you want to sort items other than
+        in alphabetical order by key.
+
+        If for_json is true (not the default), objects with a ``for_json()``
+        method will use the return value of that method for encoding as JSON
+        instead of the object.
+
+        If *ignore_nan* is true (default: ``False``), then out of range
+        :class:`float` values (``nan``, ``inf``, ``-inf``) will be serialized
+        as ``null`` in compliance with the ECMA-262 specification. If true,
+        this will override *allow_nan*.
+
+        """
+
+        self.skipkeys = skipkeys
+        self.ensure_ascii = ensure_ascii
+        self.check_circular = check_circular
+        self.allow_nan = allow_nan
+        self.sort_keys = sort_keys
+        self.use_decimal = use_decimal
+        self.namedtuple_as_object = namedtuple_as_object
+        self.tuple_as_array = tuple_as_array
+        self.iterable_as_array = iterable_as_array
+        self.bigint_as_string = bigint_as_string
+        self.item_sort_key = item_sort_key
+        self.for_json = for_json
+        self.ignore_nan = ignore_nan
+        self.int_as_string_bitcount = int_as_string_bitcount
+        if indent is not None and not isinstance(indent, string_types):
+            indent = indent * ' '
+        self.indent = indent
+        if separators is not None:
+            self.item_separator, self.key_separator = separators
+        elif indent is not None:
+            self.item_separator = ','
+        if default is not None:
+            self.default = default
+        self.encoding = encoding
+
+    def default(self, o):
+        """Implement this method in a subclass such that it returns
+        a serializable object for ``o``, or calls the base implementation
+        (to raise a ``TypeError``).
+
+        For example, to support arbitrary iterators, you could
+        implement default like this::
+
+            def default(self, o):
+                try:
+                    iterable = iter(o)
+                except TypeError:
+                    pass
+                else:
+                    return list(iterable)
+                return JSONEncoder.default(self, o)
+
+        """
+        raise TypeError('Object of type %s is not JSON serializable' %
+                        o.__class__.__name__)
+
+    def encode(self, o):
+        """Return a JSON string representation of a Python data structure.
+
+        >>> from simplejson import JSONEncoder
+        >>> JSONEncoder().encode({"foo": ["bar", "baz"]})
+        '{"foo": ["bar", "baz"]}'
+
+        """
+        # This is for extremely simple cases and benchmarks.
+        if isinstance(o, binary_type):
+            _encoding = self.encoding
+            if (_encoding is not None and not (_encoding == 'utf-8')):
+                o = text_type(o, _encoding)
+        if isinstance(o, string_types):
+            if self.ensure_ascii:
+                return encode_basestring_ascii(o)
+            else:
+                return encode_basestring(o)
+        # This doesn't pass the iterator directly to ''.join() because the
+        # exceptions aren't as detailed.  The list call should be roughly
+        # equivalent to the PySequence_Fast that ''.join() would do.
+        chunks = self.iterencode(o, _one_shot=True)
+        if not isinstance(chunks, (list, tuple)):
+            chunks = list(chunks)
+        if self.ensure_ascii:
+            return ''.join(chunks)
+        else:
+            return u''.join(chunks)
+
+    def iterencode(self, o, _one_shot=False):
+        """Encode the given object and yield each string
+        representation as available.
+
+        For example::
+
+            for chunk in JSONEncoder().iterencode(bigobject):
+                mysocket.write(chunk)
+
+        """
+        if self.check_circular:
+            markers = {}
+        else:
+            markers = None
+        if self.ensure_ascii:
+            _encoder = encode_basestring_ascii
+        else:
+            _encoder = encode_basestring
+        if self.encoding != 'utf-8' and self.encoding is not None:
+            def _encoder(o, _orig_encoder=_encoder, _encoding=self.encoding):
+                if isinstance(o, binary_type):
+                    o = text_type(o, _encoding)
+                return _orig_encoder(o)
+
+        def floatstr(o, allow_nan=self.allow_nan, ignore_nan=self.ignore_nan,
+                _repr=FLOAT_REPR, _inf=PosInf, _neginf=-PosInf):
+            # Check for specials. Note that this type of test is processor
+            # and/or platform-specific, so do tests which don't depend on
+            # the internals.
+
+            if o != o:
+                text = 'NaN'
+            elif o == _inf:
+                text = 'Infinity'
+            elif o == _neginf:
+                text = '-Infinity'
+            else:
+                if type(o) != float:
+                    # See #118, do not trust custom str/repr
+                    o = float(o)
+                return _repr(o)
+
+            if ignore_nan:
+                text = 'null'
+            elif not allow_nan:
+                raise ValueError(
+                    "Out of range float values are not JSON compliant: " +
+                    repr(o))
+
+            return text
+
+        key_memo = {}
+        int_as_string_bitcount = (
+            53 if self.bigint_as_string else self.int_as_string_bitcount)
+        if (_one_shot and c_make_encoder is not None
+                and self.indent is None):
+            _iterencode = c_make_encoder(
+                markers, self.default, _encoder, self.indent,
+                self.key_separator, self.item_separator, self.sort_keys,
+                self.skipkeys, self.allow_nan, key_memo, self.use_decimal,
+                self.namedtuple_as_object, self.tuple_as_array,
+                int_as_string_bitcount,
+                self.item_sort_key, self.encoding, self.for_json,
+                self.ignore_nan, decimal.Decimal, self.iterable_as_array)
+        else:
+            _iterencode = _make_iterencode(
+                markers, self.default, _encoder, self.indent, floatstr,
+                self.key_separator, self.item_separator, self.sort_keys,
+                self.skipkeys, _one_shot, self.use_decimal,
+                self.namedtuple_as_object, self.tuple_as_array,
+                int_as_string_bitcount,
+                self.item_sort_key, self.encoding, self.for_json,
+                self.iterable_as_array, Decimal=decimal.Decimal)
+        try:
+            return _iterencode(o, 0)
+        finally:
+            key_memo.clear()
+
+
+class JSONEncoderForHTML(JSONEncoder):
+    """An encoder that produces JSON safe to embed in HTML.
+
+    To embed JSON content in, say, a script tag on a web page, the
+    characters &, < and > should be escaped. They cannot be escaped
+    with the usual entities (e.g. &amp;) because they are not expanded
+    within <script> tags.
+
+    This class also escapes the line separator and paragraph separator
+    characters U+2028 and U+2029, irrespective of the ensure_ascii setting,
+    as these characters are not valid in JavaScript strings (see
+    http://timelessrepo.com/json-isnt-a-javascript-subset).
+    """
+
+    def encode(self, o):
+        # Override JSONEncoder.encode because it has hacks for
+        # performance that make things more complicated.
+        chunks = self.iterencode(o, True)
+        if self.ensure_ascii:
+            return ''.join(chunks)
+        else:
+            return u''.join(chunks)
+
+    def iterencode(self, o, _one_shot=False):
+        chunks = super(JSONEncoderForHTML, self).iterencode(o, _one_shot)
+        for chunk in chunks:
+            chunk = chunk.replace('&', '\\u0026')
+            chunk = chunk.replace('<', '\\u003c')
+            chunk = chunk.replace('>', '\\u003e')
+
+            if not self.ensure_ascii:
+                chunk = chunk.replace(u'\u2028', '\\u2028')
+                chunk = chunk.replace(u'\u2029', '\\u2029')
+
+            yield chunk
+
+
+def _make_iterencode(markers, _default, _encoder, _indent, _floatstr,
+        _key_separator, _item_separator, _sort_keys, _skipkeys, _one_shot,
+        _use_decimal, _namedtuple_as_object, _tuple_as_array,
+        _int_as_string_bitcount, _item_sort_key,
+        _encoding,_for_json,
+        _iterable_as_array,
+        ## HACK: hand-optimized bytecode; turn globals into locals
+        _PY3=PY3,
+        ValueError=ValueError,
+        string_types=string_types,
+        Decimal=None,
+        dict=dict,
+        float=float,
+        id=id,
+        integer_types=integer_types,
+        isinstance=isinstance,
+        list=list,
+        str=str,
+        tuple=tuple,
+        iter=iter,
+    ):
+    if _use_decimal and Decimal is None:
+        Decimal = decimal.Decimal
+    if _item_sort_key and not callable(_item_sort_key):
+        raise TypeError("item_sort_key must be None or callable")
+    elif _sort_keys and not _item_sort_key:
+        _item_sort_key = itemgetter(0)
+
+    if (_int_as_string_bitcount is not None and
+        (_int_as_string_bitcount <= 0 or
+         not isinstance(_int_as_string_bitcount, integer_types))):
+        raise TypeError("int_as_string_bitcount must be a positive integer")
+
+    def _encode_int(value):
+        skip_quoting = (
+            _int_as_string_bitcount is None
+            or
+            _int_as_string_bitcount < 1
+        )
+        if type(value) not in integer_types:
+            # See #118, do not trust custom str/repr
+            value = int(value)
+        if (
+            skip_quoting or
+            (-1 << _int_as_string_bitcount)
+            < value <
+            (1 << _int_as_string_bitcount)
+        ):
+            return str(value)
+        return '"' + str(value) + '"'
+
+    def _iterencode_list(lst, _current_indent_level):
+        if not lst:
+            yield '[]'
+            return
+        if markers is not None:
+            markerid = id(lst)
+            if markerid in markers:
+                raise ValueError("Circular reference detected")
+            markers[markerid] = lst
+        buf = '['
+        if _indent is not None:
+            _current_indent_level += 1
+            newline_indent = '\n' + (_indent * _current_indent_level)
+            separator = _item_separator + newline_indent
+            buf += newline_indent
+        else:
+            newline_indent = None
+            separator = _item_separator
+        first = True
+        for value in lst:
+            if first:
+                first = False
+            else:
+                buf = separator
+            if isinstance(value, string_types):
+                yield buf + _encoder(value)
+            elif _PY3 and isinstance(value, bytes) and _encoding is not None:
+                yield buf + _encoder(value)
+            elif isinstance(value, RawJSON):
+                yield buf + value.encoded_json
+            elif value is None:
+                yield buf + 'null'
+            elif value is True:
+                yield buf + 'true'
+            elif value is False:
+                yield buf + 'false'
+            elif isinstance(value, integer_types):
+                yield buf + _encode_int(value)
+            elif isinstance(value, float):
+                yield buf + _floatstr(value)
+            elif _use_decimal and isinstance(value, Decimal):
+                yield buf + str(value)
+            else:
+                yield buf
+                for_json = _for_json and getattr(value, 'for_json', None)
+                if for_json and callable(for_json):
+                    chunks = _iterencode(for_json(), _current_indent_level)
+                elif isinstance(value, list):
+                    chunks = _iterencode_list(value, _current_indent_level)
+                else:
+                    _asdict = _namedtuple_as_object and getattr(value, '_asdict', None)
+                    if _asdict and callable(_asdict):
+                        chunks = _iterencode_dict(_asdict(),
+                                                  _current_indent_level)
+                    elif _tuple_as_array and isinstance(value, tuple):
+                        chunks = _iterencode_list(value, _current_indent_level)
+                    elif isinstance(value, dict):
+                        chunks = _iterencode_dict(value, _current_indent_level)
+                    else:
+                        chunks = _iterencode(value, _current_indent_level)
+                for chunk in chunks:
+                    yield chunk
+        if first:
+            # iterable_as_array misses the fast path at the top
+            yield '[]'
+        else:
+            if newline_indent is not None:
+                _current_indent_level -= 1
+                yield '\n' + (_indent * _current_indent_level)
+            yield ']'
+        if markers is not None:
+            del markers[markerid]
+
+    def _stringify_key(key):
+        if isinstance(key, string_types): # pragma: no cover
+            pass
+        elif _PY3 and isinstance(key, bytes) and _encoding is not None:
+            key = str(key, _encoding)
+        elif isinstance(key, float):
+            key = _floatstr(key)
+        elif key is True:
+            key = 'true'
+        elif key is False:
+            key = 'false'
+        elif key is None:
+            key = 'null'
+        elif isinstance(key, integer_types):
+            if type(key) not in integer_types:
+                # See #118, do not trust custom str/repr
+                key = int(key)
+            key = str(key)
+        elif _use_decimal and isinstance(key, Decimal):
+            key = str(key)
+        elif _skipkeys:
+            key = None
+        else:
+            raise TypeError('keys must be str, int, float, bool or None, '
+                            'not %s' % key.__class__.__name__)
+        return key
+
+    def _iterencode_dict(dct, _current_indent_level):
+        if not dct:
+            yield '{}'
+            return
+        if markers is not None:
+            markerid = id(dct)
+            if markerid in markers:
+                raise ValueError("Circular reference detected")
+            markers[markerid] = dct
+        yield '{'
+        if _indent is not None:
+            _current_indent_level += 1
+            newline_indent = '\n' + (_indent * _current_indent_level)
+            item_separator = _item_separator + newline_indent
+            yield newline_indent
+        else:
+            newline_indent = None
+            item_separator = _item_separator
+        first = True
+        if _PY3:
+            iteritems = dct.items()
+        else:
+            iteritems = dct.iteritems()
+        if _item_sort_key:
+            items = []
+            for k, v in dct.items():
+                if not isinstance(k, string_types):
+                    k = _stringify_key(k)
+                    if k is None:
+                        continue
+                items.append((k, v))
+            items.sort(key=_item_sort_key)
+        else:
+            items = iteritems
+        for key, value in items:
+            if not (_item_sort_key or isinstance(key, string_types)):
+                key = _stringify_key(key)
+                if key is None:
+                    # _skipkeys must be True
+                    continue
+            if first:
+                first = False
+            else:
+                yield item_separator
+            yield _encoder(key)
+            yield _key_separator
+            if isinstance(value, string_types):
+                yield _encoder(value)
+            elif _PY3 and isinstance(value, bytes) and _encoding is not None:
+                yield _encoder(value)
+            elif isinstance(value, RawJSON):
+                yield value.encoded_json
+            elif value is None:
+                yield 'null'
+            elif value is True:
+                yield 'true'
+            elif value is False:
+                yield 'false'
+            elif isinstance(value, integer_types):
+                yield _encode_int(value)
+            elif isinstance(value, float):
+                yield _floatstr(value)
+            elif _use_decimal and isinstance(value, Decimal):
+                yield str(value)
+            else:
+                for_json = _for_json and getattr(value, 'for_json', None)
+                if for_json and callable(for_json):
+                    chunks = _iterencode(for_json(), _current_indent_level)
+                elif isinstance(value, list):
+                    chunks = _iterencode_list(value, _current_indent_level)
+                else:
+                    _asdict = _namedtuple_as_object and getattr(value, '_asdict', None)
+                    if _asdict and callable(_asdict):
+                        chunks = _iterencode_dict(_asdict(),
+                                                  _current_indent_level)
+                    elif _tuple_as_array and isinstance(value, tuple):
+                        chunks = _iterencode_list(value, _current_indent_level)
+                    elif isinstance(value, dict):
+                        chunks = _iterencode_dict(value, _current_indent_level)
+                    else:
+                        chunks = _iterencode(value, _current_indent_level)
+                for chunk in chunks:
+                    yield chunk
+        if newline_indent is not None:
+            _current_indent_level -= 1
+            yield '\n' + (_indent * _current_indent_level)
+        yield '}'
+        if markers is not None:
+            del markers[markerid]
+
+    def _iterencode(o, _current_indent_level):
+        if isinstance(o, string_types):
+            yield _encoder(o)
+        elif _PY3 and isinstance(o, bytes) and _encoding is not None:
+            yield _encoder(o)
+        elif isinstance(o, RawJSON):
+            yield o.encoded_json
+        elif o is None:
+            yield 'null'
+        elif o is True:
+            yield 'true'
+        elif o is False:
+            yield 'false'
+        elif isinstance(o, integer_types):
+            yield _encode_int(o)
+        elif isinstance(o, float):
+            yield _floatstr(o)
+        else:
+            for_json = _for_json and getattr(o, 'for_json', None)
+            if for_json and callable(for_json):
+                for chunk in _iterencode(for_json(), _current_indent_level):
+                    yield chunk
+            elif isinstance(o, list):
+                for chunk in _iterencode_list(o, _current_indent_level):
+                    yield chunk
+            else:
+                _asdict = _namedtuple_as_object and getattr(o, '_asdict', None)
+                if _asdict and callable(_asdict):
+                    for chunk in _iterencode_dict(_asdict(),
+                            _current_indent_level):
+                        yield chunk
+                elif (_tuple_as_array and isinstance(o, tuple)):
+                    for chunk in _iterencode_list(o, _current_indent_level):
+                        yield chunk
+                elif isinstance(o, dict):
+                    for chunk in _iterencode_dict(o, _current_indent_level):
+                        yield chunk
+                elif _use_decimal and isinstance(o, Decimal):
+                    yield str(o)
+                else:
+                    while _iterable_as_array:
+                        # Markers are not checked here because it is valid for
+                        # an iterable to return self.
+                        try:
+                            o = iter(o)
+                        except TypeError:
+                            break
+                        for chunk in _iterencode_list(o, _current_indent_level):
+                            yield chunk
+                        return
+                    if markers is not None:
+                        markerid = id(o)
+                        if markerid in markers:
+                            raise ValueError("Circular reference detected")
+                        markers[markerid] = o
+                    o = _default(o)
+                    for chunk in _iterencode(o, _current_indent_level):
+                        yield chunk
+                    if markers is not None:
+                        del markers[markerid]
+
+    return _iterencode

simplejson/errors.py

@@ -0,0 +1,53 @@
+"""Error classes used by simplejson
+"""
+__all__ = ['JSONDecodeError']
+
+
+def linecol(doc, pos):
+    lineno = doc.count('\n', 0, pos) + 1
+    if lineno == 1:
+        colno = pos + 1
+    else:
+        colno = pos - doc.rindex('\n', 0, pos)
+    return lineno, colno
+
+
+def errmsg(msg, doc, pos, end=None):
+    lineno, colno = linecol(doc, pos)
+    msg = msg.replace('%r', repr(doc[pos:pos + 1]))
+    if end is None:
+        fmt = '%s: line %d column %d (char %d)'
+        return fmt % (msg, lineno, colno, pos)
+    endlineno, endcolno = linecol(doc, end)
+    fmt = '%s: line %d column %d - line %d column %d (char %d - %d)'
+    return fmt % (msg, lineno, colno, endlineno, endcolno, pos, end)
+
+
+class JSONDecodeError(ValueError):
+    """Subclass of ValueError with the following additional properties:
+
+    msg: The unformatted error message
+    doc: The JSON document being parsed
+    pos: The start index of doc where parsing failed
+    end: The end index of doc where parsing failed (may be None)
+    lineno: The line corresponding to pos
+    colno: The column corresponding to pos
+    endlineno: The line corresponding to end (may be None)
+    endcolno: The column corresponding to end (may be None)
+
+    """
+    # Note that this exception is used from _speedups
+    def __init__(self, msg, doc, pos, end=None):
+        ValueError.__init__(self, errmsg(msg, doc, pos, end=end))
+        self.msg = msg
+        self.doc = doc
+        self.pos = pos
+        self.end = end
+        self.lineno, self.colno = linecol(doc, pos)
+        if end is not None:
+            self.endlineno, self.endcolno = linecol(doc, end)
+        else:
+            self.endlineno, self.endcolno = None, None
+
+    def __reduce__(self):
+        return self.__class__, (self.msg, self.doc, self.pos, self.end)

simplejson/ordered_dict.py

@@ -0,0 +1,103 @@
+"""Drop-in replacement for collections.OrderedDict by Raymond Hettinger
+
+http://code.activestate.com/recipes/576693/
+
+"""
+from UserDict import DictMixin
+
+class OrderedDict(dict, DictMixin):
+
+    def __init__(self, *args, **kwds):
+        if len(args) > 1:
+            raise TypeError('expected at most 1 arguments, got %d' % len(args))
+        try:
+            self.__end
+        except AttributeError:
+            self.clear()
+        self.update(*args, **kwds)
+
+    def clear(self):
+        self.__end = end = []
+        end += [None, end, end]         # sentinel node for doubly linked list
+        self.__map = {}                 # key --> [key, prev, next]
+        dict.clear(self)
+
+    def __setitem__(self, key, value):
+        if key not in self:
+            end = self.__end
+            curr = end[1]
+            curr[2] = end[1] = self.__map[key] = [key, curr, end]
+        dict.__setitem__(self, key, value)
+
+    def __delitem__(self, key):
+        dict.__delitem__(self, key)
+        key, prev, next = self.__map.pop(key)
+        prev[2] = next
+        next[1] = prev
+
+    def __iter__(self):
+        end = self.__end
+        curr = end[2]
+        while curr is not end:
+            yield curr[0]
+            curr = curr[2]
+
+    def __reversed__(self):
+        end = self.__end
+        curr = end[1]
+        while curr is not end:
+            yield curr[0]
+            curr = curr[1]
+
+    def popitem(self, last=True):
+        if not self:
+            raise KeyError('dictionary is empty')
+        key = reversed(self).next() if last else iter(self).next()
+        value = self.pop(key)
+        return key, value
+
+    def __reduce__(self):
+        items = [[k, self[k]] for k in self]
+        tmp = self.__map, self.__end
+        del self.__map, self.__end
+        inst_dict = vars(self).copy()
+        self.__map, self.__end = tmp
+        if inst_dict:
+            return (self.__class__, (items,), inst_dict)
+        return self.__class__, (items,)
+
+    def keys(self):
+        return list(self)
+
+    setdefault = DictMixin.setdefault
+    update = DictMixin.update
+    pop = DictMixin.pop
+    values = DictMixin.values
+    items = DictMixin.items
+    iterkeys = DictMixin.iterkeys
+    itervalues = DictMixin.itervalues
+    iteritems = DictMixin.iteritems
+
+    def __repr__(self):
+        if not self:
+            return '%s()' % (self.__class__.__name__,)
+        return '%s(%r)' % (self.__class__.__name__, self.items())
+
+    def copy(self):
+        return self.__class__(self)
+
+    @classmethod
+    def fromkeys(cls, iterable, value=None):
+        d = cls()
+        for key in iterable:
+            d[key] = value
+        return d
+
+    def __eq__(self, other):
+        if isinstance(other, OrderedDict):
+            return len(self)==len(other) and \
+                   all(p==q for p, q in  zip(self.items(), other.items()))
+        return dict.__eq__(self, other)
+
+    def __ne__(self, other):
+        return not self == other

simplejson/raw_json.py

@@ -0,0 +1,9 @@
+"""Implementation of RawJSON
+"""
+
+class RawJSON(object):
+    """Wrap an encoded JSON document for direct embedding in the output
+
+    """
+    def __init__(self, encoded_json):
+        self.encoded_json = encoded_json

simplejson/scanner.py

@@ -0,0 +1,85 @@
+"""JSON token scanner
+"""
+import re
+from .errors import JSONDecodeError
+def _import_c_make_scanner():
+    try:
+        from ._speedups import make_scanner
+        return make_scanner
+    except ImportError:
+        return None
+c_make_scanner = _import_c_make_scanner()
+
+__all__ = ['make_scanner', 'JSONDecodeError']
+
+NUMBER_RE = re.compile(
+    r'(-?(?:0|[1-9]\d*))(\.\d+)?([eE][-+]?\d+)?',
+    (re.VERBOSE | re.MULTILINE | re.DOTALL))
+
+
+def py_make_scanner(context):
+    parse_object = context.parse_object
+    parse_array = context.parse_array
+    parse_string = context.parse_string
+    match_number = NUMBER_RE.match
+    encoding = context.encoding
+    strict = context.strict
+    parse_float = context.parse_float
+    parse_int = context.parse_int
+    parse_constant = context.parse_constant
+    object_hook = context.object_hook
+    object_pairs_hook = context.object_pairs_hook
+    memo = context.memo
+
+    def _scan_once(string, idx):
+        errmsg = 'Expecting value'
+        try:
+            nextchar = string[idx]
+        except IndexError:
+            raise JSONDecodeError(errmsg, string, idx)
+
+        if nextchar == '"':
+            return parse_string(string, idx + 1, encoding, strict)
+        elif nextchar == '{':
+            return parse_object((string, idx + 1), encoding, strict,
+                _scan_once, object_hook, object_pairs_hook, memo)
+        elif nextchar == '[':
+            return parse_array((string, idx + 1), _scan_once)
+        elif nextchar == 'n' and string[idx:idx + 4] == 'null':
+            return None, idx + 4
+        elif nextchar == 't' and string[idx:idx + 4] == 'true':
+            return True, idx + 4
+        elif nextchar == 'f' and string[idx:idx + 5] == 'false':
+            return False, idx + 5
+
+        m = match_number(string, idx)
+        if m is not None:
+            integer, frac, exp = m.groups()
+            if frac or exp:
+                res = parse_float(integer + (frac or '') + (exp or ''))
+            else:
+                res = parse_int(integer)
+            return res, m.end()
+        elif nextchar == 'N' and string[idx:idx + 3] == 'NaN':
+            return parse_constant('NaN'), idx + 3
+        elif nextchar == 'I' and string[idx:idx + 8] == 'Infinity':
+            return parse_constant('Infinity'), idx + 8
+        elif nextchar == '-' and string[idx:idx + 9] == '-Infinity':
+            return parse_constant('-Infinity'), idx + 9
+        else:
+            raise JSONDecodeError(errmsg, string, idx)
+
+    def scan_once(string, idx):
+        if idx < 0:
+            # Ensure the same behavior as the C speedup, otherwise
+            # this would work for *some* negative string indices due
+            # to the behavior of __getitem__ for strings. #98
+            raise JSONDecodeError('Expecting value', string, idx)
+        try:
+            return _scan_once(string, idx)
+        finally:
+            memo.clear()
+
+    return scan_once
+
+make_scanner = c_make_scanner or py_make_scanner

simplejson/tests/__init__.py

@@ -0,0 +1,74 @@
+from __future__ import absolute_import
+import unittest
+import sys
+import os
+
+
+class NoExtensionTestSuite(unittest.TestSuite):
+    def run(self, result):
+        import simplejson
+        simplejson._toggle_speedups(False)
+        result = unittest.TestSuite.run(self, result)
+        simplejson._toggle_speedups(True)
+        return result
+
+
+class TestMissingSpeedups(unittest.TestCase):
+    def runTest(self):
+        if hasattr(sys, 'pypy_translation_info'):
+            "PyPy doesn't need speedups! :)"
+        elif hasattr(self, 'skipTest'):
+            self.skipTest('_speedups.so is missing!')
+
+
+def additional_tests(suite=None):
+    import simplejson
+    import simplejson.encoder
+    import simplejson.decoder
+    if suite is None:
+        suite = unittest.TestSuite()
+    try:
+        import doctest
+    except ImportError:
+        if sys.version_info < (2, 7):
+            # doctests in 2.6 depends on cStringIO
+            return suite
+        raise
+    for mod in (simplejson, simplejson.encoder, simplejson.decoder):
+        suite.addTest(doctest.DocTestSuite(mod))
+    suite.addTest(doctest.DocFileSuite('../../index.rst'))
+    return suite
+
+
+def all_tests_suite():
+    def get_suite():
+        suite_names = [
+            'simplejson.tests.%s' % (os.path.splitext(f)[0],)
+            for f in os.listdir(os.path.dirname(__file__))
+            if f.startswith('test_') and f.endswith('.py')
+        ]
+        return additional_tests(
+            unittest.TestLoader().loadTestsFromNames(suite_names))
+    suite = get_suite()
+    import simplejson
+    if simplejson._import_c_make_encoder() is None:
+        suite.addTest(TestMissingSpeedups())
+    else:
+        suite = unittest.TestSuite([
+            suite,
+            NoExtensionTestSuite([get_suite()]),
+        ])
+    return suite
+
+
+def main():
+    runner = unittest.TextTestRunner(verbosity=1 + sys.argv.count('-v'))
+    suite = all_tests_suite()
+    raise SystemExit(not runner.run(suite).wasSuccessful())
+
+
+if __name__ == '__main__':
+    import os
+    import sys
+    sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
+    main()

simplejson/tests/test_bigint_as_string.py

@@ -0,0 +1,67 @@
+from unittest import TestCase
+
+import simplejson as json
+
+
+class TestBigintAsString(TestCase):
+    # Python 2.5, at least the one that ships on Mac OS X, calculates
+    # 2 ** 53 as 0! It manages to calculate 1 << 53 correctly.
+    values = [(200, 200),
+              ((1 << 53) - 1, 9007199254740991),
+              ((1 << 53), '9007199254740992'),
+              ((1 << 53) + 1, '9007199254740993'),
+              (-100, -100),
+              ((-1 << 53), '-9007199254740992'),
+              ((-1 << 53) - 1, '-9007199254740993'),
+              ((-1 << 53) + 1, -9007199254740991)]
+
+    options = (
+        {"bigint_as_string": True},
+        {"int_as_string_bitcount": 53}
+    )
+
+    def test_ints(self):
+        for opts in self.options:
+            for val, expect in self.values:
+                self.assertEqual(
+                    val,
+                    json.loads(json.dumps(val)))
+                self.assertEqual(
+                    expect,
+                    json.loads(json.dumps(val, **opts)))
+
+    def test_lists(self):
+        for opts in self.options:
+            for val, expect in self.values:
+                val = [val, val]
+                expect = [expect, expect]
+                self.assertEqual(
+                    val,
+                    json.loads(json.dumps(val)))
+                self.assertEqual(
+                    expect,
+                    json.loads(json.dumps(val, **opts)))
+
+    def test_dicts(self):
+        for opts in self.options:
+            for val, expect in self.values:
+                val = {'k': val}
+                expect = {'k': expect}
+                self.assertEqual(
+                    val,
+                    json.loads(json.dumps(val)))
+                self.assertEqual(
+                    expect,
+                    json.loads(json.dumps(val, **opts)))
+
+    def test_dict_keys(self):
+        for opts in self.options:
+            for val, _ in self.values:
+                expect = {str(val): 'value'}
+                val = {val: 'value'}
+                self.assertEqual(
+                    expect,
+                    json.loads(json.dumps(val)))
+                self.assertEqual(
+                    expect,
+                    json.loads(json.dumps(val, **opts)))

simplejson/tests/test_bitsize_int_as_string.py

@@ -0,0 +1,73 @@
+from unittest import TestCase
+
+import simplejson as json
+
+
+class TestBitSizeIntAsString(TestCase):
+    # Python 2.5, at least the one that ships on Mac OS X, calculates
+    # 2 ** 31 as 0! It manages to calculate 1 << 31 correctly.
+    values = [
+        (200, 200),
+        ((1 << 31) - 1, (1 << 31) - 1),
+        ((1 << 31), str(1 << 31)),
+        ((1 << 31) + 1, str((1 << 31) + 1)),
+        (-100, -100),
+        ((-1 << 31), str(-1 << 31)),
+        ((-1 << 31) - 1, str((-1 << 31) - 1)),
+        ((-1 << 31) + 1, (-1 << 31) + 1),
+    ]
+
+    def test_invalid_counts(self):
+        for n in ['foo', -1, 0, 1.0]:
+            self.assertRaises(
+                TypeError,
+                json.dumps, 0, int_as_string_bitcount=n)
+
+    def test_ints_outside_range_fails(self):
+        self.assertNotEqual(
+            str(1 << 15),
+            json.loads(json.dumps(1 << 15, int_as_string_bitcount=16)),
+            )
+
+    def test_ints(self):
+        for val, expect in self.values:
+            self.assertEqual(
+                val,
+                json.loads(json.dumps(val)))
+            self.assertEqual(
+                expect,
+                json.loads(json.dumps(val, int_as_string_bitcount=31)),
+                )
+
+    def test_lists(self):
+        for val, expect in self.values:
+            val = [val, val]
+            expect = [expect, expect]
+            self.assertEqual(
+                val,
+                json.loads(json.dumps(val)))
+            self.assertEqual(
+                expect,
+                json.loads(json.dumps(val, int_as_string_bitcount=31)))
+
+    def test_dicts(self):
+        for val, expect in self.values:
+            val = {'k': val}
+            expect = {'k': expect}
+            self.assertEqual(
+                val,
+                json.loads(json.dumps(val)))
+            self.assertEqual(
+                expect,
+                json.loads(json.dumps(val, int_as_string_bitcount=31)))
+
+    def test_dict_keys(self):
+        for val, _ in self.values:
+            expect = {str(val): 'value'}
+            val = {val: 'value'}
+            self.assertEqual(
+                expect,
+                json.loads(json.dumps(val)))
+            self.assertEqual(
+                expect,
+                json.loads(json.dumps(val, int_as_string_bitcount=31)))

simplejson/tests/test_check_circular.py

@@ -0,0 +1,30 @@
+from unittest import TestCase
+import simplejson as json
+
+def default_iterable(obj):
+    return list(obj)
+
+class TestCheckCircular(TestCase):
+    def test_circular_dict(self):
+        dct = {}
+        dct['a'] = dct
+        self.assertRaises(ValueError, json.dumps, dct)
+
+    def test_circular_list(self):
+        lst = []
+        lst.append(lst)
+        self.assertRaises(ValueError, json.dumps, lst)
+
+    def test_circular_composite(self):
+        dct2 = {}
+        dct2['a'] = []
+        dct2['a'].append(dct2)
+        self.assertRaises(ValueError, json.dumps, dct2)
+
+    def test_circular_default(self):
+        json.dumps([set()], default=default_iterable)
+        self.assertRaises(TypeError, json.dumps, [set()])
+
+    def test_circular_off_default(self):
+        json.dumps([set()], default=default_iterable, check_circular=False)
+        self.assertRaises(TypeError, json.dumps, [set()], check_circular=False)

simplejson/tests/test_decimal.py

@@ -0,0 +1,71 @@
+import decimal
+from decimal import Decimal
+from unittest import TestCase
+from simplejson.compat import StringIO, reload_module
+
+import simplejson as json
+
+class TestDecimal(TestCase):
+    NUMS = "1.0", "10.00", "1.1", "1234567890.1234567890", "500"
+    def dumps(self, obj, **kw):
+        sio = StringIO()
+        json.dump(obj, sio, **kw)
+        res = json.dumps(obj, **kw)
+        self.assertEqual(res, sio.getvalue())
+        return res
+
+    def loads(self, s, **kw):
+        sio = StringIO(s)
+        res = json.loads(s, **kw)
+        self.assertEqual(res, json.load(sio, **kw))
+        return res
+
+    def test_decimal_encode(self):
+        for d in map(Decimal, self.NUMS):
+            self.assertEqual(self.dumps(d, use_decimal=True), str(d))
+
+    def test_decimal_decode(self):
+        for s in self.NUMS:
+            self.assertEqual(self.loads(s, parse_float=Decimal), Decimal(s))
+
+    def test_stringify_key(self):
+        for d in map(Decimal, self.NUMS):
+            v = {d: d}
+            self.assertEqual(
+                self.loads(
+                    self.dumps(v, use_decimal=True), parse_float=Decimal),
+                {str(d): d})
+
+    def test_decimal_roundtrip(self):
+        for d in map(Decimal, self.NUMS):
+            # The type might not be the same (int and Decimal) but they
+            # should still compare equal.
+            for v in [d, [d], {'': d}]:
+                self.assertEqual(
+                    self.loads(
+                        self.dumps(v, use_decimal=True), parse_float=Decimal),
+                    v)
+
+    def test_decimal_defaults(self):
+        d = Decimal('1.1')
+        # use_decimal=True is the default
+        self.assertRaises(TypeError, json.dumps, d, use_decimal=False)
+        self.assertEqual('1.1', json.dumps(d))
+        self.assertEqual('1.1', json.dumps(d, use_decimal=True))
+        self.assertRaises(TypeError, json.dump, d, StringIO(),
+                          use_decimal=False)
+        sio = StringIO()
+        json.dump(d, sio)
+        self.assertEqual('1.1', sio.getvalue())
+        sio = StringIO()
+        json.dump(d, sio, use_decimal=True)
+        self.assertEqual('1.1', sio.getvalue())
+
+    def test_decimal_reload(self):
+        # Simulate a subinterpreter that reloads the Python modules but not
+        # the C code https://github.com/simplejson/simplejson/issues/34
+        global Decimal
+        Decimal = reload_module(decimal).Decimal
+        import simplejson.encoder
+        simplejson.encoder.Decimal = Decimal
+        self.test_decimal_roundtrip()

simplejson/tests/test_decode.py

@@ -0,0 +1,119 @@
+from __future__ import absolute_import
+import decimal
+from unittest import TestCase
+
+import simplejson as json
+from simplejson.compat import StringIO, b, binary_type
+from simplejson import OrderedDict
+
+class MisbehavingBytesSubtype(binary_type):
+    def decode(self, encoding=None):
+        return "bad decode"
+    def __str__(self):
+        return "bad __str__"
+    def __bytes__(self):
+        return b("bad __bytes__")
+
+class TestDecode(TestCase):
+    if not hasattr(TestCase, 'assertIs'):
+        def assertIs(self, a, b):
+            self.assertTrue(a is b, '%r is %r' % (a, b))
+
+    def test_decimal(self):
+        rval = json.loads('1.1', parse_float=decimal.Decimal)
+        self.assertTrue(isinstance(rval, decimal.Decimal))
+        self.assertEqual(rval, decimal.Decimal('1.1'))
+
+    def test_float(self):
+        rval = json.loads('1', parse_int=float)
+        self.assertTrue(isinstance(rval, float))
+        self.assertEqual(rval, 1.0)
+
+    def test_decoder_optimizations(self):
+        # Several optimizations were made that skip over calls to
+        # the whitespace regex, so this test is designed to try and
+        # exercise the uncommon cases. The array cases are already covered.
+        rval = json.loads('{   "key"    :    "value"    ,  "k":"v"    }')
+        self.assertEqual(rval, {"key":"value", "k":"v"})
+
+    def test_empty_objects(self):
+        s = '{}'
+        self.assertEqual(json.loads(s), eval(s))
+        s = '[]'
+        self.assertEqual(json.loads(s), eval(s))
+        s = '""'
+        self.assertEqual(json.loads(s), eval(s))
+
+    def test_object_pairs_hook(self):
+        s = '{"xkd":1, "kcw":2, "art":3, "hxm":4, "qrt":5, "pad":6, "hoy":7}'
+        p = [("xkd", 1), ("kcw", 2), ("art", 3), ("hxm", 4),
+             ("qrt", 5), ("pad", 6), ("hoy", 7)]
+        self.assertEqual(json.loads(s), eval(s))
+        self.assertEqual(json.loads(s, object_pairs_hook=lambda x: x), p)
+        self.assertEqual(json.load(StringIO(s),
+                                   object_pairs_hook=lambda x: x), p)
+        od = json.loads(s, object_pairs_hook=OrderedDict)
+        self.assertEqual(od, OrderedDict(p))
+        self.assertEqual(type(od), OrderedDict)
+        # the object_pairs_hook takes priority over the object_hook
+        self.assertEqual(json.loads(s,
+                                    object_pairs_hook=OrderedDict,
+                                    object_hook=lambda x: None),
+                         OrderedDict(p))
+
+    def check_keys_reuse(self, source, loads):
+        rval = loads(source)
+        (a, b), (c, d) = sorted(rval[0]), sorted(rval[1])
+        self.assertIs(a, c)
+        self.assertIs(b, d)
+
+    def test_keys_reuse_str(self):
+        s = u'[{"a_key": 1, "b_\xe9": 2}, {"a_key": 3, "b_\xe9": 4}]'.encode('utf8')
+        self.check_keys_reuse(s, json.loads)
+
+    def test_keys_reuse_unicode(self):
+        s = u'[{"a_key": 1, "b_\xe9": 2}, {"a_key": 3, "b_\xe9": 4}]'
+        self.check_keys_reuse(s, json.loads)
+
+    def test_empty_strings(self):
+        self.assertEqual(json.loads('""'), "")
+        self.assertEqual(json.loads(u'""'), u"")
+        self.assertEqual(json.loads('[""]'), [""])
+        self.assertEqual(json.loads(u'[""]'), [u""])
+
+    def test_raw_decode(self):
+        cls = json.decoder.JSONDecoder
+        self.assertEqual(
+            ({'a': {}}, 9),
+            cls().raw_decode("{\"a\": {}}"))
+        # http://code.google.com/p/simplejson/issues/detail?id=85
+        self.assertEqual(
+            ({'a': {}}, 9),
+            cls(object_pairs_hook=dict).raw_decode("{\"a\": {}}"))
+        # https://github.com/simplejson/simplejson/pull/38
+        self.assertEqual(
+            ({'a': {}}, 11),
+            cls().raw_decode(" \n{\"a\": {}}"))
+
+    def test_bytes_decode(self):
+        cls = json.decoder.JSONDecoder
+        data = b('"\xe2\x82\xac"')
+        self.assertEqual(cls().decode(data), u'\u20ac')
+        self.assertEqual(cls(encoding='latin1').decode(data), u'\xe2\x82\xac')
+        self.assertEqual(cls(encoding=None).decode(data), u'\u20ac')
+
+        data = MisbehavingBytesSubtype(b('"\xe2\x82\xac"'))
+        self.assertEqual(cls().decode(data), u'\u20ac')
+        self.assertEqual(cls(encoding='latin1').decode(data), u'\xe2\x82\xac')
+        self.assertEqual(cls(encoding=None).decode(data), u'\u20ac')
+
+    def test_bounds_checking(self):
+        # https://github.com/simplejson/simplejson/issues/98
+        j = json.decoder.JSONDecoder()
+        for i in [4, 5, 6, -1, -2, -3, -4, -5, -6]:
+            self.assertRaises(ValueError, j.scan_once, '1234', i)
+            self.assertRaises(ValueError, j.raw_decode, '1234', i)
+        x, y = sorted(['128931233', '472389423'], key=id)
+        diff = id(x) - id(y)
+        self.assertRaises(ValueError, j.scan_once, y, diff)
+        self.assertRaises(ValueError, j.raw_decode, y, i)

simplejson/tests/test_default.py

@@ -0,0 +1,9 @@
+from unittest import TestCase
+
+import simplejson as json
+
+class TestDefault(TestCase):
+    def test_default(self):
+        self.assertEqual(
+            json.dumps(type, default=repr),
+            json.dumps(repr(type)))

simplejson/tests/test_dump.py

@@ -0,0 +1,249 @@
+from unittest import TestCase
+from simplejson.compat import StringIO, long_type, b, binary_type, text_type, PY3
+import simplejson as json
+
+class MisbehavingTextSubtype(text_type):
+    def __str__(self):
+        return "FAIL!"
+
+class MisbehavingBytesSubtype(binary_type):
+    def decode(self, encoding=None):
+        return "bad decode"
+    def __str__(self):
+        return "bad __str__"
+    def __bytes__(self):
+        return b("bad __bytes__")
+
+def as_text_type(s):
+    if PY3 and isinstance(s, bytes):
+        return s.decode('ascii')
+    return s
+
+def decode_iso_8859_15(b):
+    return b.decode('iso-8859-15')
+
+class TestDump(TestCase):
+    def test_dump(self):
+        sio = StringIO()
+        json.dump({}, sio)
+        self.assertEqual(sio.getvalue(), '{}')
+
+    def test_constants(self):
+        for c in [None, True, False]:
+            self.assertTrue(json.loads(json.dumps(c)) is c)
+            self.assertTrue(json.loads(json.dumps([c]))[0] is c)
+            self.assertTrue(json.loads(json.dumps({'a': c}))['a'] is c)
+
+    def test_stringify_key(self):
+        items = [(b('bytes'), 'bytes'),
+                 (1.0, '1.0'),
+                 (10, '10'),
+                 (True, 'true'),
+                 (False, 'false'),
+                 (None, 'null'),
+                 (long_type(100), '100')]
+        for k, expect in items:
+            self.assertEqual(
+                json.loads(json.dumps({k: expect})),
+                {expect: expect})
+            self.assertEqual(
+                json.loads(json.dumps({k: expect}, sort_keys=True)),
+                {expect: expect})
+        self.assertRaises(TypeError, json.dumps, {json: 1})
+        for v in [{}, {'other': 1}, {b('derp'): 1, 'herp': 2}]:
+            for sort_keys in [False, True]:
+                v0 = dict(v)
+                v0[json] = 1
+                v1 = dict((as_text_type(key), val) for (key, val) in v.items())
+                self.assertEqual(
+                    json.loads(json.dumps(v0, skipkeys=True, sort_keys=sort_keys)),
+                    v1)
+                self.assertEqual(
+                    json.loads(json.dumps({'': v0}, skipkeys=True, sort_keys=sort_keys)),
+                    {'': v1})
+                self.assertEqual(
+                    json.loads(json.dumps([v0], skipkeys=True, sort_keys=sort_keys)),
+                    [v1])
+
+    def test_dumps(self):
+        self.assertEqual(json.dumps({}), '{}')
+
+    def test_encode_truefalse(self):
+        self.assertEqual(json.dumps(
+                 {True: False, False: True}, sort_keys=True),
+                 '{"false": true, "true": false}')
+        self.assertEqual(
+            json.dumps(
+                {2: 3.0,
+                 4.0: long_type(5),
+                 False: 1,
+                 long_type(6): True,
+                 "7": 0},
+                sort_keys=True),
+            '{"2": 3.0, "4.0": 5, "6": true, "7": 0, "false": 1}')
+
+    def test_ordered_dict(self):
+        # http://bugs.python.org/issue6105
+        items = [('one', 1), ('two', 2), ('three', 3), ('four', 4), ('five', 5)]
+        s = json.dumps(json.OrderedDict(items))
+        self.assertEqual(
+            s,
+            '{"one": 1, "two": 2, "three": 3, "four": 4, "five": 5}')
+
+    def test_indent_unknown_type_acceptance(self):
+        """
+        A test against the regression mentioned at `github issue 29`_.
+
+        The indent parameter should accept any type which pretends to be
+        an instance of int or long when it comes to being multiplied by
+        strings, even if it is not actually an int or long, for
+        backwards compatibility.
+
+        .. _github issue 29:
+           http://github.com/simplejson/simplejson/issue/29
+        """
+
+        class AwesomeInt(object):
+            """An awesome reimplementation of integers"""
+
+            def __init__(self, *args, **kwargs):
+                if len(args) > 0:
+                    # [construct from literals, objects, etc.]
+                    # ...
+
+                    # Finally, if args[0] is an integer, store it
+                    if isinstance(args[0], int):
+                        self._int = args[0]
+
+            # [various methods]
+
+            def __mul__(self, other):
+                # [various ways to multiply AwesomeInt objects]
+                # ... finally, if the right-hand operand is not awesome enough,
+                # try to do a normal integer multiplication
+                if hasattr(self, '_int'):
+                    return self._int * other
+                else:
+                    raise NotImplementedError("To do non-awesome things with"
+                        " this object, please construct it from an integer!")
+
+        s = json.dumps([0, 1, 2], indent=AwesomeInt(3))
+        self.assertEqual(s, '[\n   0,\n   1,\n   2\n]')
+
+    def test_accumulator(self):
+        # the C API uses an accumulator that collects after 100,000 appends
+        lst = [0] * 100000
+        self.assertEqual(json.loads(json.dumps(lst)), lst)
+
+    def test_sort_keys(self):
+        # https://github.com/simplejson/simplejson/issues/106
+        for num_keys in range(2, 32):
+            p = dict((str(x), x) for x in range(num_keys))
+            sio = StringIO()
+            json.dump(p, sio, sort_keys=True)
+            self.assertEqual(sio.getvalue(), json.dumps(p, sort_keys=True))
+            self.assertEqual(json.loads(sio.getvalue()), p)
+
+    def test_misbehaving_text_subtype(self):
+        # https://github.com/simplejson/simplejson/issues/185
+        text = "this is some text"
+        self.assertEqual(
+            json.dumps(MisbehavingTextSubtype(text)),
+            json.dumps(text)
+        )
+        self.assertEqual(
+            json.dumps([MisbehavingTextSubtype(text)]),
+            json.dumps([text])
+        )
+        self.assertEqual(
+            json.dumps({MisbehavingTextSubtype(text): 42}),
+            json.dumps({text: 42})
+        )
+
+    def test_misbehaving_bytes_subtype(self):
+        data = b("this is some data \xe2\x82\xac")
+        self.assertEqual(
+            json.dumps(MisbehavingBytesSubtype(data)),
+            json.dumps(data)
+        )
+        self.assertEqual(
+            json.dumps([MisbehavingBytesSubtype(data)]),
+            json.dumps([data])
+        )
+        self.assertEqual(
+            json.dumps({MisbehavingBytesSubtype(data): 42}),
+            json.dumps({data: 42})
+        )
+
+    def test_bytes_toplevel(self):
+        self.assertEqual(json.dumps(b('\xe2\x82\xac')), r'"\u20ac"')
+        self.assertRaises(UnicodeDecodeError, json.dumps, b('\xa4'))
+        self.assertEqual(json.dumps(b('\xa4'), encoding='iso-8859-1'),
+                         r'"\u00a4"')
+        self.assertEqual(json.dumps(b('\xa4'), encoding='iso-8859-15'),
+                         r'"\u20ac"')
+        if PY3:
+            self.assertRaises(TypeError, json.dumps, b('\xe2\x82\xac'),
+                              encoding=None)
+            self.assertRaises(TypeError, json.dumps, b('\xa4'),
+                              encoding=None)
+            self.assertEqual(json.dumps(b('\xa4'), encoding=None,
+                                        default=decode_iso_8859_15),
+                            r'"\u20ac"')
+        else:
+            self.assertEqual(json.dumps(b('\xe2\x82\xac'), encoding=None),
+                             r'"\u20ac"')
+            self.assertRaises(UnicodeDecodeError, json.dumps, b('\xa4'),
+                              encoding=None)
+            self.assertRaises(UnicodeDecodeError, json.dumps, b('\xa4'),
+                              encoding=None, default=decode_iso_8859_15)
+
+    def test_bytes_nested(self):
+        self.assertEqual(json.dumps([b('\xe2\x82\xac')]), r'["\u20ac"]')
+        self.assertRaises(UnicodeDecodeError, json.dumps, [b('\xa4')])
+        self.assertEqual(json.dumps([b('\xa4')], encoding='iso-8859-1'),
+                         r'["\u00a4"]')
+        self.assertEqual(json.dumps([b('\xa4')], encoding='iso-8859-15'),
+                         r'["\u20ac"]')
+        if PY3:
+            self.assertRaises(TypeError, json.dumps, [b('\xe2\x82\xac')],
+                              encoding=None)
+            self.assertRaises(TypeError, json.dumps, [b('\xa4')],
+                              encoding=None)
+            self.assertEqual(json.dumps([b('\xa4')], encoding=None,
+                                        default=decode_iso_8859_15),
+                             r'["\u20ac"]')
+        else:
+            self.assertEqual(json.dumps([b('\xe2\x82\xac')], encoding=None),
+                             r'["\u20ac"]')
+            self.assertRaises(UnicodeDecodeError, json.dumps, [b('\xa4')],
+                              encoding=None)
+            self.assertRaises(UnicodeDecodeError, json.dumps, [b('\xa4')],
+                              encoding=None, default=decode_iso_8859_15)
+
+    def test_bytes_key(self):
+        self.assertEqual(json.dumps({b('\xe2\x82\xac'): 42}), r'{"\u20ac": 42}')
+        self.assertRaises(UnicodeDecodeError, json.dumps, {b('\xa4'): 42})
+        self.assertEqual(json.dumps({b('\xa4'): 42}, encoding='iso-8859-1'),
+                         r'{"\u00a4": 42}')
+        self.assertEqual(json.dumps({b('\xa4'): 42}, encoding='iso-8859-15'),
+                         r'{"\u20ac": 42}')
+        if PY3:
+            self.assertRaises(TypeError, json.dumps, {b('\xe2\x82\xac'): 42},
+                              encoding=None)
+            self.assertRaises(TypeError, json.dumps, {b('\xa4'): 42},
+                              encoding=None)
+            self.assertRaises(TypeError, json.dumps, {b('\xa4'): 42},
+                              encoding=None, default=decode_iso_8859_15)
+            self.assertEqual(json.dumps({b('\xa4'): 42}, encoding=None,
+                                        skipkeys=True),
+                             r'{}')
+        else:
+            self.assertEqual(json.dumps({b('\xe2\x82\xac'): 42}, encoding=None),
+                             r'{"\u20ac": 42}')
+            self.assertRaises(UnicodeDecodeError, json.dumps, {b('\xa4'): 42},
+                              encoding=None)
+            self.assertRaises(UnicodeDecodeError, json.dumps, {b('\xa4'): 42},
+                              encoding=None, default=decode_iso_8859_15)
+            self.assertRaises(UnicodeDecodeError, json.dumps, {b('\xa4'): 42},
+                              encoding=None, skipkeys=True)

simplejson/tests/test_encode_basestring_ascii.py

@@ -0,0 +1,47 @@
+from unittest import TestCase
+
+import simplejson.encoder
+from simplejson.compat import b
+
+CASES = [
+    (u'/\\"\ucafe\ubabe\uab98\ufcde\ubcda\uef4a\x08\x0c\n\r\t`1~!@#$%^&*()_+-=[]{}|;:\',./<>?', '"/\\\\\\"\\ucafe\\ubabe\\uab98\\ufcde\\ubcda\\uef4a\\b\\f\\n\\r\\t`1~!@#$%^&*()_+-=[]{}|;:\',./<>?"'),
+    (u'\u0123\u4567\u89ab\ucdef\uabcd\uef4a', '"\\u0123\\u4567\\u89ab\\ucdef\\uabcd\\uef4a"'),
+    (u'controls', '"controls"'),
+    (u'\x08\x0c\n\r\t', '"\\b\\f\\n\\r\\t"'),
+    (u'{"object with 1 member":["array with 1 element"]}', '"{\\"object with 1 member\\":[\\"array with 1 element\\"]}"'),
+    (u' s p a c e d ', '" s p a c e d "'),
+    (u'\U0001d120', '"\\ud834\\udd20"'),
+    (u'\u03b1\u03a9', '"\\u03b1\\u03a9"'),
+    (b('\xce\xb1\xce\xa9'), '"\\u03b1\\u03a9"'),
+    (u'\u03b1\u03a9', '"\\u03b1\\u03a9"'),
+    (b('\xce\xb1\xce\xa9'), '"\\u03b1\\u03a9"'),
+    (u'\u03b1\u03a9', '"\\u03b1\\u03a9"'),
+    (u'\u03b1\u03a9', '"\\u03b1\\u03a9"'),
+    (u"`1~!@#$%^&*()_+-={':[,]}|;.</>?", '"`1~!@#$%^&*()_+-={\':[,]}|;.</>?"'),
+    (u'\x08\x0c\n\r\t', '"\\b\\f\\n\\r\\t"'),
+    (u'\u0123\u4567\u89ab\ucdef\uabcd\uef4a', '"\\u0123\\u4567\\u89ab\\ucdef\\uabcd\\uef4a"'),
+]
+
+class TestEncodeBaseStringAscii(TestCase):
+    def test_py_encode_basestring_ascii(self):
+        self._test_encode_basestring_ascii(simplejson.encoder.py_encode_basestring_ascii)
+
+    def test_c_encode_basestring_ascii(self):
+        if not simplejson.encoder.c_encode_basestring_ascii:
+            return
+        self._test_encode_basestring_ascii(simplejson.encoder.c_encode_basestring_ascii)
+
+    def _test_encode_basestring_ascii(self, encode_basestring_ascii):
+        fname = encode_basestring_ascii.__name__
+        for input_string, expect in CASES:
+            result = encode_basestring_ascii(input_string)
+            #self.assertEqual(result, expect,
+            #    '{0!r} != {1!r} for {2}({3!r})'.format(
+            #        result, expect, fname, input_string))
+            self.assertEqual(result, expect,
+                '%r != %r for %s(%r)' % (result, expect, fname, input_string))
+
+    def test_sorted_dict(self):
+        items = [('one', 1), ('two', 2), ('three', 3), ('four', 4), ('five', 5)]
+        s = simplejson.dumps(dict(items), sort_keys=True)
+        self.assertEqual(s, '{"five": 5, "four": 4, "one": 1, "three": 3, "two": 2}')

simplejson/tests/test_encode_for_html.py

@@ -0,0 +1,38 @@
+import unittest
+
+import simplejson as json
+
+class TestEncodeForHTML(unittest.TestCase):
+
+    def setUp(self):
+        self.decoder = json.JSONDecoder()
+        self.encoder = json.JSONEncoderForHTML()
+        self.non_ascii_encoder = json.JSONEncoderForHTML(ensure_ascii=False)
+
+    def test_basic_encode(self):
+        self.assertEqual(r'"\u0026"', self.encoder.encode('&'))
+        self.assertEqual(r'"\u003c"', self.encoder.encode('<'))
+        self.assertEqual(r'"\u003e"', self.encoder.encode('>'))
+        self.assertEqual(r'"\u2028"', self.encoder.encode(u'\u2028'))
+
+    def test_non_ascii_basic_encode(self):
+        self.assertEqual(r'"\u0026"', self.non_ascii_encoder.encode('&'))
+        self.assertEqual(r'"\u003c"', self.non_ascii_encoder.encode('<'))
+        self.assertEqual(r'"\u003e"', self.non_ascii_encoder.encode('>'))
+        self.assertEqual(r'"\u2028"', self.non_ascii_encoder.encode(u'\u2028'))
+
+    def test_basic_roundtrip(self):
+        for char in '&<>':
+            self.assertEqual(
+                char, self.decoder.decode(
+                    self.encoder.encode(char)))
+
+    def test_prevent_script_breakout(self):
+        bad_string = '</script><script>alert("gotcha")</script>'
+        self.assertEqual(
+            r'"\u003c/script\u003e\u003cscript\u003e'
+            r'alert(\"gotcha\")\u003c/script\u003e"',
+            self.encoder.encode(bad_string))
+        self.assertEqual(
+            bad_string, self.decoder.decode(
+                self.encoder.encode(bad_string)))

simplejson/tests/test_errors.py

@@ -0,0 +1,68 @@
+import sys, pickle
+from unittest import TestCase
+
+import simplejson as json
+from simplejson.compat import text_type, b
+
+class TestErrors(TestCase):
+    def test_string_keys_error(self):
+        data = [{'a': 'A', 'b': (2, 4), 'c': 3.0, ('d',): 'D tuple'}]
+        try:
+            json.dumps(data)
+        except TypeError:
+            err = sys.exc_info()[1]
+        else:
+            self.fail('Expected TypeError')
+        self.assertEqual(str(err),
+                'keys must be str, int, float, bool or None, not tuple')
+
+    def test_not_serializable(self):
+        try:
+            json.dumps(json)
+        except TypeError:
+            err = sys.exc_info()[1]
+        else:
+            self.fail('Expected TypeError')
+        self.assertEqual(str(err),
+                'Object of type module is not JSON serializable')
+
+    def test_decode_error(self):
+        err = None
+        try:
+            json.loads('{}\na\nb')
+        except json.JSONDecodeError:
+            err = sys.exc_info()[1]
+        else:
+            self.fail('Expected JSONDecodeError')
+        self.assertEqual(err.lineno, 2)
+        self.assertEqual(err.colno, 1)
+        self.assertEqual(err.endlineno, 3)
+        self.assertEqual(err.endcolno, 2)
+
+    def test_scan_error(self):
+        err = None
+        for t in (text_type, b):
+            try:
+                json.loads(t('{"asdf": "'))
+            except json.JSONDecodeError:
+                err = sys.exc_info()[1]
+            else:
+                self.fail('Expected JSONDecodeError')
+            self.assertEqual(err.lineno, 1)
+            self.assertEqual(err.colno, 10)
+
+    def test_error_is_pickable(self):
+        err = None
+        try:
+            json.loads('{}\na\nb')
+        except json.JSONDecodeError:
+            err = sys.exc_info()[1]
+        else:
+            self.fail('Expected JSONDecodeError')
+        s = pickle.dumps(err)
+        e = pickle.loads(s)
+
+        self.assertEqual(err.msg, e.msg)
+        self.assertEqual(err.doc, e.doc)
+        self.assertEqual(err.pos, e.pos)
+        self.assertEqual(err.end, e.end)

simplejson/tests/test_fail.py

@@ -0,0 +1,176 @@
+import sys
+from unittest import TestCase
+
+import simplejson as json
+
+# 2007-10-05
+JSONDOCS = [
+    # http://json.org/JSON_checker/test/fail1.json
+    '"A JSON payload should be an object or array, not a string."',
+    # http://json.org/JSON_checker/test/fail2.json
+    '["Unclosed array"',
+    # http://json.org/JSON_checker/test/fail3.json
+    '{unquoted_key: "keys must be quoted"}',
+    # http://json.org/JSON_checker/test/fail4.json
+    '["extra comma",]',
+    # http://json.org/JSON_checker/test/fail5.json
+    '["double extra comma",,]',
+    # http://json.org/JSON_checker/test/fail6.json
+    '[   , "<-- missing value"]',
+    # http://json.org/JSON_checker/test/fail7.json
+    '["Comma after the close"],',
+    # http://json.org/JSON_checker/test/fail8.json
+    '["Extra close"]]',
+    # http://json.org/JSON_checker/test/fail9.json
+    '{"Extra comma": true,}',
+    # http://json.org/JSON_checker/test/fail10.json
+    '{"Extra value after close": true} "misplaced quoted value"',
+    # http://json.org/JSON_checker/test/fail11.json
+    '{"Illegal expression": 1 + 2}',
+    # http://json.org/JSON_checker/test/fail12.json
+    '{"Illegal invocation": alert()}',
+    # http://json.org/JSON_checker/test/fail13.json
+    '{"Numbers cannot have leading zeroes": 013}',
+    # http://json.org/JSON_checker/test/fail14.json
+    '{"Numbers cannot be hex": 0x14}',
+    # http://json.org/JSON_checker/test/fail15.json
+    '["Illegal backslash escape: \\x15"]',
+    # http://json.org/JSON_checker/test/fail16.json
+    '[\\naked]',
+    # http://json.org/JSON_checker/test/fail17.json
+    '["Illegal backslash escape: \\017"]',
+    # http://json.org/JSON_checker/test/fail18.json
+    '[[[[[[[[[[[[[[[[[[[["Too deep"]]]]]]]]]]]]]]]]]]]]',
+    # http://json.org/JSON_checker/test/fail19.json
+    '{"Missing colon" null}',
+    # http://json.org/JSON_checker/test/fail20.json
+    '{"Double colon":: null}',
+    # http://json.org/JSON_checker/test/fail21.json
+    '{"Comma instead of colon", null}',
+    # http://json.org/JSON_checker/test/fail22.json
+    '["Colon instead of comma": false]',
+    # http://json.org/JSON_checker/test/fail23.json
+    '["Bad value", truth]',
+    # http://json.org/JSON_checker/test/fail24.json
+    "['single quote']",
+    # http://json.org/JSON_checker/test/fail25.json
+    '["\ttab\tcharacter\tin\tstring\t"]',
+    # http://json.org/JSON_checker/test/fail26.json
+    '["tab\\   character\\   in\\  string\\  "]',
+    # http://json.org/JSON_checker/test/fail27.json
+    '["line\nbreak"]',
+    # http://json.org/JSON_checker/test/fail28.json
+    '["line\\\nbreak"]',
+    # http://json.org/JSON_checker/test/fail29.json
+    '[0e]',
+    # http://json.org/JSON_checker/test/fail30.json
+    '[0e+]',
+    # http://json.org/JSON_checker/test/fail31.json
+    '[0e+-1]',
+    # http://json.org/JSON_checker/test/fail32.json
+    '{"Comma instead if closing brace": true,',
+    # http://json.org/JSON_checker/test/fail33.json
+    '["mismatch"}',
+    # http://code.google.com/p/simplejson/issues/detail?id=3
+    u'["A\u001FZ control characters in string"]',
+    # misc based on coverage
+    '{',
+    '{]',
+    '{"foo": "bar"]',
+    '{"foo": "bar"',
+    'nul',
+    'nulx',
+    '-',
+    '-x',
+    '-e',
+    '-e0',
+    '-Infinite',
+    '-Inf',
+    'Infinit',
+    'Infinite',
+    'NaM',
+    'NuN',
+    'falsy',
+    'fal',
+    'trug',
+    'tru',
+    '1e',
+    '1ex',
+    '1e-',
+    '1e-x',
+]
+
+SKIPS = {
+    1: "why not have a string payload?",
+    18: "spec doesn't specify any nesting limitations",
+}
+
+class TestFail(TestCase):
+    def test_failures(self):
+        for idx, doc in enumerate(JSONDOCS):
+            idx = idx + 1
+            if idx in SKIPS:
+                json.loads(doc)
+                continue
+            try:
+                json.loads(doc)
+            except json.JSONDecodeError:
+                pass
+            else:
+                self.fail("Expected failure for fail%d.json: %r" % (idx, doc))
+
+    def test_array_decoder_issue46(self):
+        # http://code.google.com/p/simplejson/issues/detail?id=46
+        for doc in [u'[,]', '[,]']:
+            try:
+                json.loads(doc)
+            except json.JSONDecodeError:
+                e = sys.exc_info()[1]
+                self.assertEqual(e.pos, 1)
+                self.assertEqual(e.lineno, 1)
+                self.assertEqual(e.colno, 2)
+            except Exception:
+                e = sys.exc_info()[1]
+                self.fail("Unexpected exception raised %r %s" % (e, e))
+            else:
+                self.fail("Unexpected success parsing '[,]'")
+
+    def test_truncated_input(self):
+        test_cases = [
+            ('', 'Expecting value', 0),
+            ('[', "Expecting value or ']'", 1),
+            ('[42', "Expecting ',' delimiter", 3),
+            ('[42,', 'Expecting value', 4),
+            ('["', 'Unterminated string starting at', 1),
+            ('["spam', 'Unterminated string starting at', 1),
+            ('["spam"', "Expecting ',' delimiter", 7),
+            ('["spam",', 'Expecting value', 8),
+            ('{', 'Expecting property name enclosed in double quotes', 1),
+            ('{"', 'Unterminated string starting at', 1),
+            ('{"spam', 'Unterminated string starting at', 1),
+            ('{"spam"', "Expecting ':' delimiter", 7),
+            ('{"spam":', 'Expecting value', 8),
+            ('{"spam":42', "Expecting ',' delimiter", 10),
+            ('{"spam":42,', 'Expecting property name enclosed in double quotes',
+             11),
+            ('"', 'Unterminated string starting at', 0),
+            ('"spam', 'Unterminated string starting at', 0),
+            ('[,', "Expecting value", 1),
+        ]
+        for data, msg, idx in test_cases:
+            try:
+                json.loads(data)
+            except json.JSONDecodeError:
+                e = sys.exc_info()[1]
+                self.assertEqual(
+                    e.msg[:len(msg)],
+                    msg,
+                    "%r doesn't start with %r for %r" % (e.msg, msg, data))
+                self.assertEqual(
+                    e.pos, idx,
+                    "pos %r != %r for %r" % (e.pos, idx, data))
+            except Exception:
+                e = sys.exc_info()[1]
+                self.fail("Unexpected exception raised %r %s" % (e, e))
+            else:
+                self.fail("Unexpected success parsing '%r'" % (data,))

simplejson/tests/test_float.py

@@ -0,0 +1,35 @@
+import math
+from unittest import TestCase
+from simplejson.compat import long_type, text_type
+import simplejson as json
+from simplejson.decoder import NaN, PosInf, NegInf
+
+class TestFloat(TestCase):
+    def test_degenerates_allow(self):
+        for inf in (PosInf, NegInf):
+            self.assertEqual(json.loads(json.dumps(inf)), inf)
+        # Python 2.5 doesn't have math.isnan
+        nan = json.loads(json.dumps(NaN))
+        self.assertTrue((0 + nan) != nan)
+
+    def test_degenerates_ignore(self):
+        for f in (PosInf, NegInf, NaN):
+            self.assertEqual(json.loads(json.dumps(f, ignore_nan=True)), None)
+
+    def test_degenerates_deny(self):
+        for f in (PosInf, NegInf, NaN):
+            self.assertRaises(ValueError, json.dumps, f, allow_nan=False)
+
+    def test_floats(self):
+        for num in [1617161771.7650001, math.pi, math.pi**100,
+                    math.pi**-100, 3.1]:
+            self.assertEqual(float(json.dumps(num)), num)
+            self.assertEqual(json.loads(json.dumps(num)), num)
+            self.assertEqual(json.loads(text_type(json.dumps(num))), num)
+
+    def test_ints(self):
+        for num in [1, long_type(1), 1<<32, 1<<64]:
+            self.assertEqual(json.dumps(num), str(num))
+            self.assertEqual(int(json.dumps(num)), num)
+            self.assertEqual(json.loads(json.dumps(num)), num)
+            self.assertEqual(json.loads(text_type(json.dumps(num))), num)

simplejson/tests/test_for_json.py

@@ -0,0 +1,97 @@
+import unittest
+import simplejson as json
+
+
+class ForJson(object):
+    def for_json(self):
+        return {'for_json': 1}
+
+
+class NestedForJson(object):
+    def for_json(self):
+        return {'nested': ForJson()}
+
+
+class ForJsonList(object):
+    def for_json(self):
+        return ['list']
+
+
+class DictForJson(dict):
+    def for_json(self):
+        return {'alpha': 1}
+
+
+class ListForJson(list):
+    def for_json(self):
+        return ['list']
+
+
+class TestForJson(unittest.TestCase):
+    def assertRoundTrip(self, obj, other, for_json=True):
+        if for_json is None:
+            # None will use the default
+            s = json.dumps(obj)
+        else:
+            s = json.dumps(obj, for_json=for_json)
+        self.assertEqual(
+            json.loads(s),
+            other)
+
+    def test_for_json_encodes_stand_alone_object(self):
+        self.assertRoundTrip(
+            ForJson(),
+            ForJson().for_json())
+
+    def test_for_json_encodes_object_nested_in_dict(self):
+        self.assertRoundTrip(
+            {'hooray': ForJson()},
+            {'hooray': ForJson().for_json()})
+
+    def test_for_json_encodes_object_nested_in_list_within_dict(self):
+        self.assertRoundTrip(
+            {'list': [0, ForJson(), 2, 3]},
+            {'list': [0, ForJson().for_json(), 2, 3]})
+
+    def test_for_json_encodes_object_nested_within_object(self):
+        self.assertRoundTrip(
+            NestedForJson(),
+            {'nested': {'for_json': 1}})
+
+    def test_for_json_encodes_list(self):
+        self.assertRoundTrip(
+            ForJsonList(),
+            ForJsonList().for_json())
+
+    def test_for_json_encodes_list_within_object(self):
+        self.assertRoundTrip(
+            {'nested': ForJsonList()},
+            {'nested': ForJsonList().for_json()})
+
+    def test_for_json_encodes_dict_subclass(self):
+        self.assertRoundTrip(
+            DictForJson(a=1),
+            DictForJson(a=1).for_json())
+
+    def test_for_json_encodes_list_subclass(self):
+        self.assertRoundTrip(
+            ListForJson(['l']),
+            ListForJson(['l']).for_json())
+
+    def test_for_json_ignored_if_not_true_with_dict_subclass(self):
+        for for_json in (None, False):
+            self.assertRoundTrip(
+                DictForJson(a=1),
+                {'a': 1},
+                for_json=for_json)
+
+    def test_for_json_ignored_if_not_true_with_list_subclass(self):
+        for for_json in (None, False):
+            self.assertRoundTrip(
+                ListForJson(['l']),
+                ['l'],
+                for_json=for_json)
+
+    def test_raises_typeerror_if_for_json_not_true_with_object(self):
+        self.assertRaises(TypeError, json.dumps, ForJson())
+        self.assertRaises(TypeError, json.dumps, ForJson(), for_json=False)

simplejson/tests/test_indent.py

@@ -0,0 +1,86 @@
+from unittest import TestCase
+import textwrap
+
+import simplejson as json
+from simplejson.compat import StringIO
+
+class TestIndent(TestCase):
+    def test_indent(self):
+        h = [['blorpie'], ['whoops'], [], 'd-shtaeou', 'd-nthiouh',
+             'i-vhbjkhnth',
+             {'nifty': 87}, {'field': 'yes', 'morefield': False} ]
+
+        expect = textwrap.dedent("""\
+        [
+        \t[
+        \t\t"blorpie"
+        \t],
+        \t[
+        \t\t"whoops"
+        \t],
+        \t[],
+        \t"d-shtaeou",
+        \t"d-nthiouh",
+        \t"i-vhbjkhnth",
+        \t{
+        \t\t"nifty": 87
+        \t},
+        \t{
+        \t\t"field": "yes",
+        \t\t"morefield": false
+        \t}
+        ]""")
+
+
+        d1 = json.dumps(h)
+        d2 = json.dumps(h, indent='\t', sort_keys=True, separators=(',', ': '))
+        d3 = json.dumps(h, indent='  ', sort_keys=True, separators=(',', ': '))
+        d4 = json.dumps(h, indent=2, sort_keys=True, separators=(',', ': '))
+
+        h1 = json.loads(d1)
+        h2 = json.loads(d2)
+        h3 = json.loads(d3)
+        h4 = json.loads(d4)
+
+        self.assertEqual(h1, h)
+        self.assertEqual(h2, h)
+        self.assertEqual(h3, h)
+        self.assertEqual(h4, h)
+        self.assertEqual(d3, expect.replace('\t', '  '))
+        self.assertEqual(d4, expect.replace('\t', '  '))
+        # NOTE: Python 2.4 textwrap.dedent converts tabs to spaces,
+        #       so the following is expected to fail. Python 2.4 is not a
+        #       supported platform in simplejson 2.1.0+.
+        self.assertEqual(d2, expect)
+
+    def test_indent0(self):
+        h = {3: 1}
+        def check(indent, expected):
+            d1 = json.dumps(h, indent=indent)
+            self.assertEqual(d1, expected)
+
+            sio = StringIO()
+            json.dump(h, sio, indent=indent)
+            self.assertEqual(sio.getvalue(), expected)
+
+        # indent=0 should emit newlines
+        check(0, '{\n"3": 1\n}')
+        # indent=None is more compact
+        check(None, '{"3": 1}')
+
+    def test_separators(self):
+        lst = [1,2,3,4]
+        expect = '[\n1,\n2,\n3,\n4\n]'
+        expect_spaces = '[\n1, \n2, \n3, \n4\n]'
+        # Ensure that separators still works
+        self.assertEqual(
+            expect_spaces,
+            json.dumps(lst, indent=0, separators=(', ', ': ')))
+        # Force the new defaults
+        self.assertEqual(
+            expect,
+            json.dumps(lst, indent=0, separators=(',', ': ')))
+        # Added in 2.1.4
+        self.assertEqual(
+            expect,
+            json.dumps(lst, indent=0))

simplejson/tests/test_item_sort_key.py

@@ -0,0 +1,27 @@
+from unittest import TestCase
+
+import simplejson as json
+from operator import itemgetter
+
+class TestItemSortKey(TestCase):
+    def test_simple_first(self):
+        a = {'a': 1, 'c': 5, 'jack': 'jill', 'pick': 'axe', 'array': [1, 5, 6, 9], 'tuple': (83, 12, 3), 'crate': 'dog', 'zeak': 'oh'}
+        self.assertEqual(
+            '{"a": 1, "c": 5, "crate": "dog", "jack": "jill", "pick": "axe", "zeak": "oh", "array": [1, 5, 6, 9], "tuple": [83, 12, 3]}',
+            json.dumps(a, item_sort_key=json.simple_first))
+
+    def test_case(self):
+        a = {'a': 1, 'c': 5, 'Jack': 'jill', 'pick': 'axe', 'Array': [1, 5, 6, 9], 'tuple': (83, 12, 3), 'crate': 'dog', 'zeak': 'oh'}
+        self.assertEqual(
+            '{"Array": [1, 5, 6, 9], "Jack": "jill", "a": 1, "c": 5, "crate": "dog", "pick": "axe", "tuple": [83, 12, 3], "zeak": "oh"}',
+            json.dumps(a, item_sort_key=itemgetter(0)))
+        self.assertEqual(
+            '{"a": 1, "Array": [1, 5, 6, 9], "c": 5, "crate": "dog", "Jack": "jill", "pick": "axe", "tuple": [83, 12, 3], "zeak": "oh"}',
+            json.dumps(a, item_sort_key=lambda kv: kv[0].lower()))
+
+    def test_item_sort_key_value(self):
+        # https://github.com/simplejson/simplejson/issues/173
+        a = {'a': 1, 'b': 0}
+        self.assertEqual(
+            '{"b": 0, "a": 1}',
+            json.dumps(a, item_sort_key=lambda kv: kv[1]))

simplejson/tests/test_iterable.py

@@ -0,0 +1,31 @@
+import unittest
+from simplejson.compat import StringIO
+
+import simplejson as json
+
+def iter_dumps(obj, **kw):
+    return ''.join(json.JSONEncoder(**kw).iterencode(obj))
+
+def sio_dump(obj, **kw):
+    sio = StringIO()
+    json.dumps(obj, **kw)
+    return sio.getvalue()
+
+class TestIterable(unittest.TestCase):
+    def test_iterable(self):
+        for l in ([], [1], [1, 2], [1, 2, 3]):
+            for opts in [{}, {'indent': 2}]:
+                for dumps in (json.dumps, iter_dumps, sio_dump):
+                    expect = dumps(l, **opts)
+                    default_expect = dumps(sum(l), **opts)
+                    # Default is False
+                    self.assertRaises(TypeError, dumps, iter(l), **opts)
+                    self.assertRaises(TypeError, dumps, iter(l), iterable_as_array=False, **opts)
+                    self.assertEqual(expect, dumps(iter(l), iterable_as_array=True, **opts))
+                    # Ensure that the "default" gets called
+                    self.assertEqual(default_expect, dumps(iter(l), default=sum, **opts))
+                    self.assertEqual(default_expect, dumps(iter(l), iterable_as_array=False, default=sum, **opts))
+                    # Ensure that the "default" does not get called
+                    self.assertEqual(
+                        expect,
+                        dumps(iter(l), iterable_as_array=True, default=sum, **opts))

simplejson/tests/test_namedtuple.py

@@ -0,0 +1,122 @@
+from __future__ import absolute_import
+import unittest
+import simplejson as json
+from simplejson.compat import StringIO
+
+try:
+    from collections import namedtuple
+except ImportError:
+    class Value(tuple):
+        def __new__(cls, *args):
+            return tuple.__new__(cls, args)
+
+        def _asdict(self):
+            return {'value': self[0]}
+    class Point(tuple):
+        def __new__(cls, *args):
+            return tuple.__new__(cls, args)
+
+        def _asdict(self):
+            return {'x': self[0], 'y': self[1]}
+else:
+    Value = namedtuple('Value', ['value'])
+    Point = namedtuple('Point', ['x', 'y'])
+
+class DuckValue(object):
+    def __init__(self, *args):
+        self.value = Value(*args)
+
+    def _asdict(self):
+        return self.value._asdict()
+
+class DuckPoint(object):
+    def __init__(self, *args):
+        self.point = Point(*args)
+
+    def _asdict(self):
+        return self.point._asdict()
+
+class DeadDuck(object):
+    _asdict = None
+
+class DeadDict(dict):
+    _asdict = None
+
+CONSTRUCTORS = [
+    lambda v: v,
+    lambda v: [v],
+    lambda v: [{'key': v}],
+]
+
+class TestNamedTuple(unittest.TestCase):
+    def test_namedtuple_dumps(self):
+        for v in [Value(1), Point(1, 2), DuckValue(1), DuckPoint(1, 2)]:
+            d = v._asdict()
+            self.assertEqual(d, json.loads(json.dumps(v)))
+            self.assertEqual(
+                d,
+                json.loads(json.dumps(v, namedtuple_as_object=True)))
+            self.assertEqual(d, json.loads(json.dumps(v, tuple_as_array=False)))
+            self.assertEqual(
+                d,
+                json.loads(json.dumps(v, namedtuple_as_object=True,
+                                      tuple_as_array=False)))
+
+    def test_namedtuple_dumps_false(self):
+        for v in [Value(1), Point(1, 2)]:
+            l = list(v)
+            self.assertEqual(
+                l,
+                json.loads(json.dumps(v, namedtuple_as_object=False)))
+            self.assertRaises(TypeError, json.dumps, v,
+                tuple_as_array=False, namedtuple_as_object=False)
+
+    def test_namedtuple_dump(self):
+        for v in [Value(1), Point(1, 2), DuckValue(1), DuckPoint(1, 2)]:
+            d = v._asdict()
+            sio = StringIO()
+            json.dump(v, sio)
+            self.assertEqual(d, json.loads(sio.getvalue()))
+            sio = StringIO()
+            json.dump(v, sio, namedtuple_as_object=True)
+            self.assertEqual(
+                d,
+                json.loads(sio.getvalue()))
+            sio = StringIO()
+            json.dump(v, sio, tuple_as_array=False)
+            self.assertEqual(d, json.loads(sio.getvalue()))
+            sio = StringIO()
+            json.dump(v, sio, namedtuple_as_object=True,
+                      tuple_as_array=False)
+            self.assertEqual(
+                d,
+                json.loads(sio.getvalue()))
+
+    def test_namedtuple_dump_false(self):
+        for v in [Value(1), Point(1, 2)]:
+            l = list(v)
+            sio = StringIO()
+            json.dump(v, sio, namedtuple_as_object=False)
+            self.assertEqual(
+                l,
+                json.loads(sio.getvalue()))
+            self.assertRaises(TypeError, json.dump, v, StringIO(),
+                tuple_as_array=False, namedtuple_as_object=False)
+
+    def test_asdict_not_callable_dump(self):
+        for f in CONSTRUCTORS:
+            self.assertRaises(TypeError,
+                json.dump, f(DeadDuck()), StringIO(), namedtuple_as_object=True)
+            sio = StringIO()
+            json.dump(f(DeadDict()), sio, namedtuple_as_object=True)
+            self.assertEqual(
+                json.dumps(f({})),
+                sio.getvalue())
+
+    def test_asdict_not_callable_dumps(self):
+        for f in CONSTRUCTORS:
+            self.assertRaises(TypeError,
+                json.dumps, f(DeadDuck()), namedtuple_as_object=True)
+            self.assertEqual(
+                json.dumps(f({})),
+                json.dumps(f(DeadDict()), namedtuple_as_object=True))

simplejson/tests/test_pass1.py

@@ -0,0 +1,71 @@
+from unittest import TestCase
+
+import simplejson as json
+
+# from http://json.org/JSON_checker/test/pass1.json
+JSON = r'''
+[
+    "JSON Test Pattern pass1",
+    {"object with 1 member":["array with 1 element"]},
+    {},
+    [],
+    -42,
+    true,
+    false,
+    null,
+    {
+        "integer": 1234567890,
+        "real": -9876.543210,
+        "e": 0.123456789e-12,
+        "E": 1.234567890E+34,
+        "":  23456789012E66,
+        "zero": 0,
+        "one": 1,
+        "space": " ",
+        "quote": "\"",
+        "backslash": "\\",
+        "controls": "\b\f\n\r\t",
+        "slash": "/ & \/",
+        "alpha": "abcdefghijklmnopqrstuvwyz",
+        "ALPHA": "ABCDEFGHIJKLMNOPQRSTUVWYZ",
+        "digit": "0123456789",
+        "special": "`1~!@#$%^&*()_+-={':[,]}|;.</>?",
+        "hex": "\u0123\u4567\u89AB\uCDEF\uabcd\uef4A",
+        "true": true,
+        "false": false,
+        "null": null,
+        "array":[  ],
+        "object":{  },
+        "address": "50 St. James Street",
+        "url": "http://www.JSON.org/",
+        "comment": "// /* <!-- --",
+        "# -- --> */": " ",
+        " s p a c e d " :[1,2 , 3
+
+,
+
+4 , 5        ,          6           ,7        ],"compact": [1,2,3,4,5,6,7],
+        "jsontext": "{\"object with 1 member\":[\"array with 1 element\"]}",
+        "quotes": "&#34; \u0022 %22 0x22 034 &#x22;",
+        "\/\\\"\uCAFE\uBABE\uAB98\uFCDE\ubcda\uef4A\b\f\n\r\t`1~!@#$%^&*()_+-=[]{}|;:',./<>?"
+: "A key can be any string"
+    },
+    0.5 ,98.6
+,
+99.44
+,
+
+1066,
+1e1,
+0.1e1,
+1e-1,
+1e00,2e+00,2e-00
+,"rosebud"]
+'''
+
+class TestPass1(TestCase):
+    def test_parse(self):
+        # test in/out equivalence and parsing
+        res = json.loads(JSON)
+        out = json.dumps(res)
+        self.assertEqual(res, json.loads(out))

simplejson/tests/test_pass2.py

@@ -0,0 +1,14 @@
+from unittest import TestCase
+import simplejson as json
+
+# from http://json.org/JSON_checker/test/pass2.json
+JSON = r'''
+[[[[[[[[[[[[[[[[[[["Not too deep"]]]]]]]]]]]]]]]]]]]
+'''
+
+class TestPass2(TestCase):
+    def test_parse(self):
+        # test in/out equivalence and parsing
+        res = json.loads(JSON)
+        out = json.dumps(res)
+        self.assertEqual(res, json.loads(out))

simplejson/tests/test_pass3.py

@@ -0,0 +1,20 @@
+from unittest import TestCase
+
+import simplejson as json
+
+# from http://json.org/JSON_checker/test/pass3.json
+JSON = r'''
+{
+    "JSON Test Pattern pass3": {
+        "The outermost value": "must be an object or array.",
+        "In this test": "It is an object."
+    }
+}
+'''
+
+class TestPass3(TestCase):
+    def test_parse(self):
+        # test in/out equivalence and parsing
+        res = json.loads(JSON)
+        out = json.dumps(res)
+        self.assertEqual(res, json.loads(out))

simplejson/tests/test_raw_json.py

@@ -0,0 +1,47 @@
+import unittest
+import simplejson as json
+
+dct1 = {
+    'key1': 'value1'
+}
+
+dct2 = {
+    'key2': 'value2',
+    'd1': dct1
+}
+
+dct3 = {
+    'key2': 'value2',
+    'd1': json.dumps(dct1)
+}
+
+dct4 = {
+    'key2': 'value2',
+    'd1': json.RawJSON(json.dumps(dct1))
+}
+
+
+class TestRawJson(unittest.TestCase):
+
+    def test_normal_str(self):
+        self.assertNotEqual(json.dumps(dct2), json.dumps(dct3))
+
+    def test_raw_json_str(self):
+        self.assertEqual(json.dumps(dct2), json.dumps(dct4))
+        self.assertEqual(dct2, json.loads(json.dumps(dct4)))
+
+    def test_list(self):
+        self.assertEqual(
+            json.dumps([dct2]),
+            json.dumps([json.RawJSON(json.dumps(dct2))]))
+        self.assertEqual(
+            [dct2],
+            json.loads(json.dumps([json.RawJSON(json.dumps(dct2))])))
+
+    def test_direct(self):
+        self.assertEqual(
+            json.dumps(dct2),
+            json.dumps(json.RawJSON(json.dumps(dct2))))
+        self.assertEqual(
+            dct2,
+            json.loads(json.dumps(json.RawJSON(json.dumps(dct2)))))

simplejson/tests/test_recursion.py

@@ -0,0 +1,67 @@
+from unittest import TestCase
+
+import simplejson as json
+
+class JSONTestObject:
+    pass
+
+
+class RecursiveJSONEncoder(json.JSONEncoder):
+    recurse = False
+    def default(self, o):
+        if o is JSONTestObject:
+            if self.recurse:
+                return [JSONTestObject]
+            else:
+                return 'JSONTestObject'
+        return json.JSONEncoder.default(o)
+
+
+class TestRecursion(TestCase):
+    def test_listrecursion(self):
+        x = []
+        x.append(x)
+        try:
+            json.dumps(x)
+        except ValueError:
+            pass
+        else:
+            self.fail("didn't raise ValueError on list recursion")
+        x = []
+        y = [x]
+        x.append(y)
+        try:
+            json.dumps(x)
+        except ValueError:
+            pass
+        else:
+            self.fail("didn't raise ValueError on alternating list recursion")
+        y = []
+        x = [y, y]
+        # ensure that the marker is cleared
+        json.dumps(x)
+
+    def test_dictrecursion(self):
+        x = {}
+        x["test"] = x
+        try:
+            json.dumps(x)
+        except ValueError:
+            pass
+        else:
+            self.fail("didn't raise ValueError on dict recursion")
+        x = {}
+        y = {"a": x, "b": x}
+        # ensure that the marker is cleared
+        json.dumps(y)
+
+    def test_defaultrecursion(self):
+        enc = RecursiveJSONEncoder()
+        self.assertEqual(enc.encode(JSONTestObject), '"JSONTestObject"')
+        enc.recurse = True
+        try:
+            enc.encode(JSONTestObject)
+        except ValueError:
+            pass
+        else:
+            self.fail("didn't raise ValueError on default recursion")

simplejson/tests/test_scanstring.py

@@ -0,0 +1,196 @@
+import sys
+from unittest import TestCase
+
+import simplejson as json
+import simplejson.decoder
+from simplejson.compat import b, PY3
+
+class TestScanString(TestCase):
+    # The bytes type is intentionally not used in most of these tests
+    # under Python 3 because the decoder immediately coerces to str before
+    # calling scanstring. In Python 2 we are testing the code paths
+    # for both unicode and str.
+    #
+    # The reason this is done is because Python 3 would require
+    # entirely different code paths for parsing bytes and str.
+    #
+    def test_py_scanstring(self):
+        self._test_scanstring(simplejson.decoder.py_scanstring)
+
+    def test_c_scanstring(self):
+        if not simplejson.decoder.c_scanstring:
+            return
+        self._test_scanstring(simplejson.decoder.c_scanstring)
+
+        self.assertTrue(isinstance(simplejson.decoder.c_scanstring('""', 0)[0], str))
+
+    def _test_scanstring(self, scanstring):
+        if sys.maxunicode == 65535:
+            self.assertEqual(
+                scanstring(u'"z\U0001d120x"', 1, None, True),
+                (u'z\U0001d120x', 6))
+        else:
+            self.assertEqual(
+                scanstring(u'"z\U0001d120x"', 1, None, True),
+                (u'z\U0001d120x', 5))
+
+        self.assertEqual(
+            scanstring('"\\u007b"', 1, None, True),
+            (u'{', 8))
+
+        self.assertEqual(
+            scanstring('"A JSON payload should be an object or array, not a string."', 1, None, True),
+            (u'A JSON payload should be an object or array, not a string.', 60))
+
+        self.assertEqual(
+            scanstring('["Unclosed array"', 2, None, True),
+            (u'Unclosed array', 17))
+
+        self.assertEqual(
+            scanstring('["extra comma",]', 2, None, True),
+            (u'extra comma', 14))
+
+        self.assertEqual(
+            scanstring('["double extra comma",,]', 2, None, True),
+            (u'double extra comma', 21))
+
+        self.assertEqual(
+            scanstring('["Comma after the close"],', 2, None, True),
+            (u'Comma after the close', 24))
+
+        self.assertEqual(
+            scanstring('["Extra close"]]', 2, None, True),
+            (u'Extra close', 14))
+
+        self.assertEqual(
+            scanstring('{"Extra comma": true,}', 2, None, True),
+            (u'Extra comma', 14))
+
+        self.assertEqual(
+            scanstring('{"Extra value after close": true} "misplaced quoted value"', 2, None, True),
+            (u'Extra value after close', 26))
+
+        self.assertEqual(
+            scanstring('{"Illegal expression": 1 + 2}', 2, None, True),
+            (u'Illegal expression', 21))
+
+        self.assertEqual(
+            scanstring('{"Illegal invocation": alert()}', 2, None, True),
+            (u'Illegal invocation', 21))
+
+        self.assertEqual(
+            scanstring('{"Numbers cannot have leading zeroes": 013}', 2, None, True),
+            (u'Numbers cannot have leading zeroes', 37))
+
+        self.assertEqual(
+            scanstring('{"Numbers cannot be hex": 0x14}', 2, None, True),
+            (u'Numbers cannot be hex', 24))
+
+        self.assertEqual(
+            scanstring('[[[[[[[[[[[[[[[[[[[["Too deep"]]]]]]]]]]]]]]]]]]]]', 21, None, True),
+            (u'Too deep', 30))
+
+        self.assertEqual(
+            scanstring('{"Missing colon" null}', 2, None, True),
+            (u'Missing colon', 16))
+
+        self.assertEqual(
+            scanstring('{"Double colon":: null}', 2, None, True),
+            (u'Double colon', 15))
+
+        self.assertEqual(
+            scanstring('{"Comma instead of colon", null}', 2, None, True),
+            (u'Comma instead of colon', 25))
+
+        self.assertEqual(
+            scanstring('["Colon instead of comma": false]', 2, None, True),
+            (u'Colon instead of comma', 25))
+
+        self.assertEqual(
+            scanstring('["Bad value", truth]', 2, None, True),
+            (u'Bad value', 12))
+
+        for c in map(chr, range(0x00, 0x1f)):
+            self.assertEqual(
+                scanstring(c + '"', 0, None, False),
+                (c, 2))
+            self.assertRaises(
+                ValueError,
+                scanstring, c + '"', 0, None, True)
+
+        self.assertRaises(ValueError, scanstring, '', 0, None, True)
+        self.assertRaises(ValueError, scanstring, 'a', 0, None, True)
+        self.assertRaises(ValueError, scanstring, '\\', 0, None, True)
+        self.assertRaises(ValueError, scanstring, '\\u', 0, None, True)
+        self.assertRaises(ValueError, scanstring, '\\u0', 0, None, True)
+        self.assertRaises(ValueError, scanstring, '\\u01', 0, None, True)
+        self.assertRaises(ValueError, scanstring, '\\u012', 0, None, True)
+        self.assertRaises(ValueError, scanstring, '\\u0123', 0, None, True)
+        if sys.maxunicode > 65535:
+            self.assertRaises(ValueError,
+                              scanstring, '\\ud834\\u"', 0, None, True)
+            self.assertRaises(ValueError,
+                              scanstring, '\\ud834\\x0123"', 0, None, True)
+
+    def test_issue3623(self):
+        self.assertRaises(ValueError, json.decoder.scanstring, "xxx", 1,
+                          "xxx")
+        self.assertRaises(UnicodeDecodeError,
+                          json.encoder.encode_basestring_ascii, b("xx\xff"))
+
+    def test_overflow(self):
+        # Python 2.5 does not have maxsize, Python 3 does not have maxint
+        maxsize = getattr(sys, 'maxsize', getattr(sys, 'maxint', None))
+        assert maxsize is not None
+        self.assertRaises(OverflowError, json.decoder.scanstring, "xxx",
+                          maxsize + 1)
+
+    def test_surrogates(self):
+        scanstring = json.decoder.scanstring
+
+        def assertScan(given, expect, test_utf8=True):
+            givens = [given]
+            if not PY3 and test_utf8:
+                givens.append(given.encode('utf8'))
+            for given in givens:
+                (res, count) = scanstring(given, 1, None, True)
+                self.assertEqual(len(given), count)
+                self.assertEqual(res, expect)
+
+        assertScan(
+            u'"z\\ud834\\u0079x"',
+            u'z\ud834yx')
+        assertScan(
+            u'"z\\ud834\\udd20x"',
+            u'z\U0001d120x')
+        assertScan(
+            u'"z\\ud834\\ud834\\udd20x"',
+            u'z\ud834\U0001d120x')
+        assertScan(
+            u'"z\\ud834x"',
+            u'z\ud834x')
+        assertScan(
+            u'"z\\udd20x"',
+            u'z\udd20x')
+        assertScan(
+            u'"z\ud834x"',
+            u'z\ud834x')
+        # It may look strange to join strings together, but Python is drunk.
+        # https://gist.github.com/etrepum/5538443
+        assertScan(
+            u'"z\\ud834\udd20x12345"',
+            u''.join([u'z\ud834', u'\udd20x12345']))
+        assertScan(
+            u'"z\ud834\\udd20x"',
+            u''.join([u'z\ud834', u'\udd20x']))
+        # these have different behavior given UTF8 input, because the surrogate
+        # pair may be joined (in maxunicode > 65535 builds)
+        assertScan(
+            u''.join([u'"z\ud834', u'\udd20x"']),
+            u''.join([u'z\ud834', u'\udd20x']),
+            test_utf8=False)
+
+        self.assertRaises(ValueError,
+                          scanstring, u'"z\\ud83x"', 1, None, True)
+        self.assertRaises(ValueError,
+                          scanstring, u'"z\\ud834\\udd2x"', 1, None, True)

simplejson/tests/test_separators.py

@@ -0,0 +1,42 @@
+import textwrap
+from unittest import TestCase
+
+import simplejson as json
+
+
+class TestSeparators(TestCase):
+    def test_separators(self):
+        h = [['blorpie'], ['whoops'], [], 'd-shtaeou', 'd-nthiouh', 'i-vhbjkhnth',
+             {'nifty': 87}, {'field': 'yes', 'morefield': False} ]
+
+        expect = textwrap.dedent("""\
+        [
+          [
+            "blorpie"
+          ] ,
+          [
+            "whoops"
+          ] ,
+          [] ,
+          "d-shtaeou" ,
+          "d-nthiouh" ,
+          "i-vhbjkhnth" ,
+          {
+            "nifty" : 87
+          } ,
+          {
+            "field" : "yes" ,
+            "morefield" : false
+          }
+        ]""")
+
+
+        d1 = json.dumps(h)
+        d2 = json.dumps(h, indent='  ', sort_keys=True, separators=(' ,', ' : '))
+
+        h1 = json.loads(d1)
+        h2 = json.loads(d2)
+
+        self.assertEqual(h1, h)
+        self.assertEqual(h2, h)
+        self.assertEqual(d2, expect)

simplejson/tests/test_speedups.py

@@ -0,0 +1,114 @@
+from __future__ import with_statement
+
+import sys
+import unittest
+from unittest import TestCase
+
+import simplejson
+from simplejson import encoder, decoder, scanner
+from simplejson.compat import PY3, long_type, b
+
+
+def has_speedups():
+    return encoder.c_make_encoder is not None
+
+
+def skip_if_speedups_missing(func):
+    def wrapper(*args, **kwargs):
+        if not has_speedups():
+            if hasattr(unittest, 'SkipTest'):
+                raise unittest.SkipTest("C Extension not available")
+            else:
+                sys.stdout.write("C Extension not available")
+                return
+        return func(*args, **kwargs)
+
+    return wrapper
+
+
+class BadBool:
+    def __bool__(self):
+        1/0
+    __nonzero__ = __bool__
+
+
+class TestDecode(TestCase):
+    @skip_if_speedups_missing
+    def test_make_scanner(self):
+        self.assertRaises(AttributeError, scanner.c_make_scanner, 1)
+
+    @skip_if_speedups_missing
+    def test_bad_bool_args(self):
+        def test(value):
+            decoder.JSONDecoder(strict=BadBool()).decode(value)
+        self.assertRaises(ZeroDivisionError, test, '""')
+        self.assertRaises(ZeroDivisionError, test, '{}')
+        if not PY3:
+            self.assertRaises(ZeroDivisionError, test, u'""')
+            self.assertRaises(ZeroDivisionError, test, u'{}')
+
+class TestEncode(TestCase):
+    @skip_if_speedups_missing
+    def test_make_encoder(self):
+        self.assertRaises(
+            TypeError,
+            encoder.c_make_encoder,
+            None,
+            ("\xCD\x7D\x3D\x4E\x12\x4C\xF9\x79\xD7"
+             "\x52\xBA\x82\xF2\x27\x4A\x7D\xA0\xCA\x75"),
+            None
+        )
+
+    @skip_if_speedups_missing
+    def test_bad_str_encoder(self):
+        # Issue #31505: There shouldn't be an assertion failure in case
+        # c_make_encoder() receives a bad encoder() argument.
+        import decimal
+        def bad_encoder1(*args):
+            return None
+        enc = encoder.c_make_encoder(
+                None, lambda obj: str(obj),
+                bad_encoder1, None, ': ', ', ',
+                False, False, False, {}, False, False, False,
+                None, None, 'utf-8', False, False, decimal.Decimal, False)
+        self.assertRaises(TypeError, enc, 'spam', 4)
+        self.assertRaises(TypeError, enc, {'spam': 42}, 4)
+
+        def bad_encoder2(*args):
+            1/0
+        enc = encoder.c_make_encoder(
+                None, lambda obj: str(obj),
+                bad_encoder2, None, ': ', ', ',
+                False, False, False, {}, False, False, False,
+                None, None, 'utf-8', False, False, decimal.Decimal, False)
+        self.assertRaises(ZeroDivisionError, enc, 'spam', 4)
+
+    @skip_if_speedups_missing
+    def test_bad_bool_args(self):
+        def test(name):
+            encoder.JSONEncoder(**{name: BadBool()}).encode({})
+        self.assertRaises(ZeroDivisionError, test, 'skipkeys')
+        self.assertRaises(ZeroDivisionError, test, 'ensure_ascii')
+        self.assertRaises(ZeroDivisionError, test, 'check_circular')
+        self.assertRaises(ZeroDivisionError, test, 'allow_nan')
+        self.assertRaises(ZeroDivisionError, test, 'sort_keys')
+        self.assertRaises(ZeroDivisionError, test, 'use_decimal')
+        self.assertRaises(ZeroDivisionError, test, 'namedtuple_as_object')
+        self.assertRaises(ZeroDivisionError, test, 'tuple_as_array')
+        self.assertRaises(ZeroDivisionError, test, 'bigint_as_string')
+        self.assertRaises(ZeroDivisionError, test, 'for_json')
+        self.assertRaises(ZeroDivisionError, test, 'ignore_nan')
+        self.assertRaises(ZeroDivisionError, test, 'iterable_as_array')
+
+    @skip_if_speedups_missing
+    def test_int_as_string_bitcount_overflow(self):
+        long_count = long_type(2)**32+31
+        def test():
+            encoder.JSONEncoder(int_as_string_bitcount=long_count).encode(0)
+        self.assertRaises((TypeError, OverflowError), test)
+
+    if PY3:
+        @skip_if_speedups_missing
+        def test_bad_encoding(self):
+            with self.assertRaises(UnicodeEncodeError):
+                encoder.JSONEncoder(encoding='\udcff').encode({b('key'): 123})

simplejson/tests/test_str_subclass.py

@@ -0,0 +1,21 @@
+from unittest import TestCase
+
+import simplejson
+from simplejson.compat import text_type
+
+# Tests for issue demonstrated in https://github.com/simplejson/simplejson/issues/144
+class WonkyTextSubclass(text_type):
+    def __getslice__(self, start, end):
+        return self.__class__('not what you wanted!')
+
+class TestStrSubclass(TestCase):
+    def test_dump_load(self):
+        for s in ['', '"hello"', 'text', u'\u005c']:
+            self.assertEqual(
+                s,
+                simplejson.loads(simplejson.dumps(WonkyTextSubclass(s))))
+
+            self.assertEqual(
+                s,
+                simplejson.loads(simplejson.dumps(WonkyTextSubclass(s),
+                                                  ensure_ascii=False)))

simplejson/tests/test_subclass.py

@@ -0,0 +1,37 @@
+from unittest import TestCase
+import simplejson as json
+
+from decimal import Decimal
+
+class AlternateInt(int):
+    def __repr__(self):
+        return 'invalid json'
+    __str__ = __repr__
+
+
+class AlternateFloat(float):
+    def __repr__(self):
+        return 'invalid json'
+    __str__ = __repr__
+
+
+# class AlternateDecimal(Decimal):
+#     def __repr__(self):
+#         return 'invalid json'
+
+
+class TestSubclass(TestCase):
+    def test_int(self):
+        self.assertEqual(json.dumps(AlternateInt(1)), '1')
+        self.assertEqual(json.dumps(AlternateInt(-1)), '-1')
+        self.assertEqual(json.loads(json.dumps({AlternateInt(1): 1})), {'1': 1})
+
+    def test_float(self):
+        self.assertEqual(json.dumps(AlternateFloat(1.0)), '1.0')
+        self.assertEqual(json.dumps(AlternateFloat(-1.0)), '-1.0')
+        self.assertEqual(json.loads(json.dumps({AlternateFloat(1.0): 1})), {'1.0': 1})
+
+    # NOTE: Decimal subclasses are not supported as-is
+    # def test_decimal(self):
+    #     self.assertEqual(json.dumps(AlternateDecimal('1.0')), '1.0')
+    #     self.assertEqual(json.dumps(AlternateDecimal('-1.0')), '-1.0')

simplejson/tests/test_tool.py

@@ -0,0 +1,114 @@
+from __future__ import with_statement
+import os
+import sys
+import textwrap
+import unittest
+import subprocess
+import tempfile
+try:
+    # Python 3.x
+    from test.support import strip_python_stderr
+except ImportError:
+    # Python 2.6+
+    try:
+        from test.test_support import strip_python_stderr
+    except ImportError:
+        # Python 2.5
+        import re
+        def strip_python_stderr(stderr):
+            return re.sub(
+                r"\[\d+ refs\]\r?\n?$".encode(),
+                "".encode(),
+                stderr).strip()
+
+def open_temp_file():
+    if sys.version_info >= (2, 6):
+        file = tempfile.NamedTemporaryFile(delete=False)
+        filename = file.name
+    else:
+        fd, filename = tempfile.mkstemp()
+        file = os.fdopen(fd, 'w+b')
+    return file, filename
+
+class TestTool(unittest.TestCase):
+    data = """
+
+        [["blorpie"],[ "whoops" ] , [
+                                 ],\t"d-shtaeou",\r"d-nthiouh",
+        "i-vhbjkhnth", {"nifty":87}, {"morefield" :\tfalse,"field"
+            :"yes"}  ]
+           """
+
+    expect = textwrap.dedent("""\
+    [
+        [
+            "blorpie"
+        ],
+        [
+            "whoops"
+        ],
+        [],
+        "d-shtaeou",
+        "d-nthiouh",
+        "i-vhbjkhnth",
+        {
+            "nifty": 87
+        },
+        {
+            "field": "yes",
+            "morefield": false
+        }
+    ]
+    """)
+
+    def runTool(self, args=None, data=None):
+        argv = [sys.executable, '-m', 'simplejson.tool']
+        if args:
+            argv.extend(args)
+        proc = subprocess.Popen(argv,
+                                stdin=subprocess.PIPE,
+                                stderr=subprocess.PIPE,
+                                stdout=subprocess.PIPE)
+        out, err = proc.communicate(data)
+        self.assertEqual(strip_python_stderr(err), ''.encode())
+        self.assertEqual(proc.returncode, 0)
+        return out.decode('utf8').splitlines()
+
+    def test_stdin_stdout(self):
+        self.assertEqual(
+            self.runTool(data=self.data.encode()),
+            self.expect.splitlines())
+
+    def test_infile_stdout(self):
+        infile, infile_name = open_temp_file()
+        try:
+            infile.write(self.data.encode())
+            infile.close()
+            self.assertEqual(
+                self.runTool(args=[infile_name]),
+                self.expect.splitlines())
+        finally:
+            os.unlink(infile_name)
+
+    def test_infile_outfile(self):
+        infile, infile_name = open_temp_file()
+        try:
+            infile.write(self.data.encode())
+            infile.close()
+            # outfile will get overwritten by tool, so the delete
+            # may not work on some platforms. Do it manually.
+            outfile, outfile_name = open_temp_file()
+            try:
+                outfile.close()
+                self.assertEqual(
+                    self.runTool(args=[infile_name, outfile_name]),
+                    [])
+                with open(outfile_name, 'rb') as f:
+                    self.assertEqual(
+                        f.read().decode('utf8').splitlines(),
+                        self.expect.splitlines()
+                    )
+            finally:
+                os.unlink(outfile_name)
+        finally:
+            os.unlink(infile_name)

simplejson/tests/test_tuple.py

@@ -0,0 +1,47 @@
+import unittest
+
+from simplejson.compat import StringIO
+import simplejson as json
+
+class TestTuples(unittest.TestCase):
+    def test_tuple_array_dumps(self):
+        t = (1, 2, 3)
+        expect = json.dumps(list(t))
+        # Default is True
+        self.assertEqual(expect, json.dumps(t))
+        self.assertEqual(expect, json.dumps(t, tuple_as_array=True))
+        self.assertRaises(TypeError, json.dumps, t, tuple_as_array=False)
+        # Ensure that the "default" does not get called
+        self.assertEqual(expect, json.dumps(t, default=repr))
+        self.assertEqual(expect, json.dumps(t, tuple_as_array=True,
+                                            default=repr))
+        # Ensure that the "default" gets called
+        self.assertEqual(
+            json.dumps(repr(t)),
+            json.dumps(t, tuple_as_array=False, default=repr))
+
+    def test_tuple_array_dump(self):
+        t = (1, 2, 3)
+        expect = json.dumps(list(t))
+        # Default is True
+        sio = StringIO()
+        json.dump(t, sio)
+        self.assertEqual(expect, sio.getvalue())
+        sio = StringIO()
+        json.dump(t, sio, tuple_as_array=True)
+        self.assertEqual(expect, sio.getvalue())
+        self.assertRaises(TypeError, json.dump, t, StringIO(),
+                          tuple_as_array=False)
+        # Ensure that the "default" does not get called
+        sio = StringIO()
+        json.dump(t, sio, default=repr)
+        self.assertEqual(expect, sio.getvalue())
+        sio = StringIO()
+        json.dump(t, sio, tuple_as_array=True, default=repr)
+        self.assertEqual(expect, sio.getvalue())
+        # Ensure that the "default" gets called
+        sio = StringIO()
+        json.dump(t, sio, tuple_as_array=False, default=repr)
+        self.assertEqual(
+            json.dumps(repr(t)),
+            sio.getvalue())

simplejson/tests/test_unicode.py

@@ -0,0 +1,154 @@
+import sys
+import codecs
+from unittest import TestCase
+
+import simplejson as json
+from simplejson.compat import unichr, text_type, b, BytesIO
+
+class TestUnicode(TestCase):
+    def test_encoding1(self):
+        encoder = json.JSONEncoder(encoding='utf-8')
+        u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}'
+        s = u.encode('utf-8')
+        ju = encoder.encode(u)
+        js = encoder.encode(s)
+        self.assertEqual(ju, js)
+
+    def test_encoding2(self):
+        u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}'
+        s = u.encode('utf-8')
+        ju = json.dumps(u, encoding='utf-8')
+        js = json.dumps(s, encoding='utf-8')
+        self.assertEqual(ju, js)
+
+    def test_encoding3(self):
+        u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}'
+        j = json.dumps(u)
+        self.assertEqual(j, '"\\u03b1\\u03a9"')
+
+    def test_encoding4(self):
+        u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}'
+        j = json.dumps([u])
+        self.assertEqual(j, '["\\u03b1\\u03a9"]')
+
+    def test_encoding5(self):
+        u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}'
+        j = json.dumps(u, ensure_ascii=False)
+        self.assertEqual(j, u'"' + u + u'"')
+
+    def test_encoding6(self):
+        u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}'
+        j = json.dumps([u], ensure_ascii=False)
+        self.assertEqual(j, u'["' + u + u'"]')
+
+    def test_big_unicode_encode(self):
+        u = u'\U0001d120'
+        self.assertEqual(json.dumps(u), '"\\ud834\\udd20"')
+        self.assertEqual(json.dumps(u, ensure_ascii=False), u'"\U0001d120"')
+
+    def test_big_unicode_decode(self):
+        u = u'z\U0001d120x'
+        self.assertEqual(json.loads('"' + u + '"'), u)
+        self.assertEqual(json.loads('"z\\ud834\\udd20x"'), u)
+
+    def test_unicode_decode(self):
+        for i in range(0, 0xd7ff):
+            u = unichr(i)
+            #s = '"\\u{0:04x}"'.format(i)
+            s = '"\\u%04x"' % (i,)
+            self.assertEqual(json.loads(s), u)
+
+    def test_object_pairs_hook_with_unicode(self):
+        s = u'{"xkd":1, "kcw":2, "art":3, "hxm":4, "qrt":5, "pad":6, "hoy":7}'
+        p = [(u"xkd", 1), (u"kcw", 2), (u"art", 3), (u"hxm", 4),
+             (u"qrt", 5), (u"pad", 6), (u"hoy", 7)]
+        self.assertEqual(json.loads(s), eval(s))
+        self.assertEqual(json.loads(s, object_pairs_hook=lambda x: x), p)
+        od = json.loads(s, object_pairs_hook=json.OrderedDict)
+        self.assertEqual(od, json.OrderedDict(p))
+        self.assertEqual(type(od), json.OrderedDict)
+        # the object_pairs_hook takes priority over the object_hook
+        self.assertEqual(json.loads(s,
+                                    object_pairs_hook=json.OrderedDict,
+                                    object_hook=lambda x: None),
+                         json.OrderedDict(p))
+
+
+    def test_default_encoding(self):
+        self.assertEqual(json.loads(u'{"a": "\xe9"}'.encode('utf-8')),
+            {'a': u'\xe9'})
+
+    def test_unicode_preservation(self):
+        self.assertEqual(type(json.loads(u'""')), text_type)
+        self.assertEqual(type(json.loads(u'"a"')), text_type)
+        self.assertEqual(type(json.loads(u'["a"]')[0]), text_type)
+
+    def test_ensure_ascii_false_returns_unicode(self):
+        # http://code.google.com/p/simplejson/issues/detail?id=48
+        self.assertEqual(type(json.dumps([], ensure_ascii=False)), text_type)
+        self.assertEqual(type(json.dumps(0, ensure_ascii=False)), text_type)
+        self.assertEqual(type(json.dumps({}, ensure_ascii=False)), text_type)
+        self.assertEqual(type(json.dumps("", ensure_ascii=False)), text_type)
+
+    def test_ensure_ascii_false_bytestring_encoding(self):
+        # http://code.google.com/p/simplejson/issues/detail?id=48
+        doc1 = {u'quux': b('Arr\xc3\xaat sur images')}
+        doc2 = {u'quux': u'Arr\xeat sur images'}
+        doc_ascii = '{"quux": "Arr\\u00eat sur images"}'
+        doc_unicode = u'{"quux": "Arr\xeat sur images"}'
+        self.assertEqual(json.dumps(doc1), doc_ascii)
+        self.assertEqual(json.dumps(doc2), doc_ascii)
+        self.assertEqual(json.dumps(doc1, ensure_ascii=False), doc_unicode)
+        self.assertEqual(json.dumps(doc2, ensure_ascii=False), doc_unicode)
+
+    def test_ensure_ascii_linebreak_encoding(self):
+        # http://timelessrepo.com/json-isnt-a-javascript-subset
+        s1 = u'\u2029\u2028'
+        s2 = s1.encode('utf8')
+        expect = '"\\u2029\\u2028"'
+        expect_non_ascii = u'"\u2029\u2028"'
+        self.assertEqual(json.dumps(s1), expect)
+        self.assertEqual(json.dumps(s2), expect)
+        self.assertEqual(json.dumps(s1, ensure_ascii=False), expect_non_ascii)
+        self.assertEqual(json.dumps(s2, ensure_ascii=False), expect_non_ascii)
+
+    def test_invalid_escape_sequences(self):
+        # incomplete escape sequence
+        self.assertRaises(json.JSONDecodeError, json.loads, '"\\u')
+        self.assertRaises(json.JSONDecodeError, json.loads, '"\\u1')
+        self.assertRaises(json.JSONDecodeError, json.loads, '"\\u12')
+        self.assertRaises(json.JSONDecodeError, json.loads, '"\\u123')
+        self.assertRaises(json.JSONDecodeError, json.loads, '"\\u1234')
+        # invalid escape sequence
+        self.assertRaises(json.JSONDecodeError, json.loads, '"\\u123x"')
+        self.assertRaises(json.JSONDecodeError, json.loads, '"\\u12x4"')
+        self.assertRaises(json.JSONDecodeError, json.loads, '"\\u1x34"')
+        self.assertRaises(json.JSONDecodeError, json.loads, '"\\ux234"')
+        if sys.maxunicode > 65535:
+            # invalid escape sequence for low surrogate
+            self.assertRaises(json.JSONDecodeError, json.loads, '"\\ud800\\u"')
+            self.assertRaises(json.JSONDecodeError, json.loads, '"\\ud800\\u0"')
+            self.assertRaises(json.JSONDecodeError, json.loads, '"\\ud800\\u00"')
+            self.assertRaises(json.JSONDecodeError, json.loads, '"\\ud800\\u000"')
+            self.assertRaises(json.JSONDecodeError, json.loads, '"\\ud800\\u000x"')
+            self.assertRaises(json.JSONDecodeError, json.loads, '"\\ud800\\u00x0"')
+            self.assertRaises(json.JSONDecodeError, json.loads, '"\\ud800\\u0x00"')
+            self.assertRaises(json.JSONDecodeError, json.loads, '"\\ud800\\ux000"')
+
+    def test_ensure_ascii_still_works(self):
+        # in the ascii range, ensure that everything is the same
+        for c in map(unichr, range(0, 127)):
+            self.assertEqual(
+                json.dumps(c, ensure_ascii=False),
+                json.dumps(c))
+        snowman = u'\N{SNOWMAN}'
+        self.assertEqual(
+            json.dumps(c, ensure_ascii=False),
+            '"' + c + '"')
+
+    def test_strip_bom(self):
+        content = u"\u3053\u3093\u306b\u3061\u308f"
+        json_doc = codecs.BOM_UTF8 + b(json.dumps(content))
+        self.assertEqual(json.load(BytesIO(json_doc)), content)
+        for doc in json_doc, json_doc.decode('utf8'):
+            self.assertEqual(json.loads(doc), content)

simplejson/tool.py

@@ -0,0 +1,42 @@
+r"""Command-line tool to validate and pretty-print JSON
+
+Usage::
+
+    $ echo '{"json":"obj"}' | python -m simplejson.tool
+    {
+        "json": "obj"
+    }
+    $ echo '{ 1.2:3.4}' | python -m simplejson.tool
+    Expecting property name: line 1 column 2 (char 2)
+
+"""
+from __future__ import with_statement
+import sys
+import simplejson as json
+
+def main():
+    if len(sys.argv) == 1:
+        infile = sys.stdin
+        outfile = sys.stdout
+    elif len(sys.argv) == 2:
+        infile = open(sys.argv[1], 'r')
+        outfile = sys.stdout
+    elif len(sys.argv) == 3:
+        infile = open(sys.argv[1], 'r')
+        outfile = open(sys.argv[2], 'w')
+    else:
+        raise SystemExit(sys.argv[0] + " [infile [outfile]]")
+    with infile:
+        try:
+            obj = json.load(infile,
+                            object_pairs_hook=json.OrderedDict,
+                            use_decimal=True)
+        except ValueError:
+            raise SystemExit(sys.exc_info()[1])
+    with outfile:
+        json.dump(obj, outfile, sort_keys=True, indent='    ', use_decimal=True)
+        outfile.write('\n')
+
+
+if __name__ == '__main__':
+    main()

test/correct-results-008.txt

@@ -1,4 +1,4 @@
-Flawfinder version 2.0.15, (C) 2001-2019 David A. Wheeler.
+Flawfinder version 2.0.19, (C) 2001-2019 David A. Wheeler.
 Showing hits not in test-saved-hitlist-008.txt
 Number of rules (primarily dangerous function names) in C/C++ ruleset: 222
 

test/correct-results.csv

@@ -1,40 +1,40 @@
-File,Line,Column,Level,Category,Name,Warning,Suggestion,Note,CWEs,Context,Fingerprint
-test.c,32,2,5,buffer,gets,"Does not check for buffer overflows (CWE-120, CWE-20)",Use fgets() instead,,"CWE-120, CWE-20", gets(f);,6a5bb383fb44030b0d9428b17359e94ba3979bc1ce702be450427f85592c649a
-test.c,60,3,5,buffer,strncat,"Easily used incorrectly (e.g., incorrectly computing the correct maximum size to add) [MS-banned] (CWE-120)","Consider strcat_s, strlcat, snprintf, or automatically resizing strings","Risk is high; the length parameter appears to be a constant, instead of computing the number of characters left.",CWE-120,"  strncat(d,s,sizeof(d)); /* Misuse - this should be flagged as riskier. */",cbd19c308547e79af13436d8f7dbcf6c62e49e4f62ba9aee38fbef29e0772f74
-test.c,61,3,5,buffer,_tcsncat,"Easily used incorrectly (e.g., incorrectly computing the correct maximum size to add) [MS-banned] (CWE-120)","Consider strcat_s, strlcat, or automatically resizing strings","Risk is high; the length parameter appears to be a constant, instead of computing the number of characters left.",CWE-120,"  _tcsncat(d,s,sizeof(d)); /* Misuse - flag as riskier */",c3f6ba2c710efc878e66df4578894fd408452cb7cdec7ae6f492a3b1796f8c42
-test.c,64,3,5,buffer,MultiByteToWideChar,"Requires maximum length in CHARACTERS, not bytes (CWE-120)",,"Risk is high, it appears that the size is given as bytes, but the function requires size as characters.",CWE-120,"  MultiByteToWideChar(CP_ACP,0,szName,-1,wszUserName,sizeof(wszUserName));",4f5b73ff337a54d6e1d9a369659ca0ddb4f80e6b7e38a17e5b112f6d3e266e69
-test.c,66,3,5,buffer,MultiByteToWideChar,"Requires maximum length in CHARACTERS, not bytes (CWE-120)",,"Risk is high, it appears that the size is given as bytes, but the function requires size as characters.",CWE-120,"  MultiByteToWideChar(CP_ACP,0,szName,-1,wszUserName,sizeof wszUserName);",9ecdc1e903acc16a646bf7909a630ae22a7593b70952c39ce6bd9c5a23fad0fd
-test.c,77,3,5,misc,SetSecurityDescriptorDacl,"Never create NULL ACLs; an attacker can set it to Everyone (Deny All Access), which would even forbid administrator access (CWE-732)",,,CWE-732,"  SetSecurityDescriptorDacl(&sd,TRUE,NULL,FALSE);",5fed1e135b593b4c943e66e89a26ff131eba18b83a32a8af37d1c0bd7b01aadb
-test.c,77,3,5,misc,SetSecurityDescriptorDacl,"Never create NULL ACLs; an attacker can set it to Everyone (Deny All Access), which would even forbid administrator access (CWE-732)",,,CWE-732,"  SetSecurityDescriptorDacl(&sd,TRUE,NULL,FALSE);",5fed1e135b593b4c943e66e89a26ff131eba18b83a32a8af37d1c0bd7b01aadb
-test.c,17,2,4,buffer,strcpy,Does not check for buffer overflows when copying to destination [MS-banned] (CWE-120),"Consider using snprintf, strcpy_s, or strlcpy (warning: strncpy easily misused)",,CWE-120," strcpy(b, a);",c01c8472bb53022e912da4da2faebc67d537855da324020c44bfd5e608a79b77
-test.c,20,2,4,buffer,sprintf,Does not check for buffer overflows (CWE-120),"Use sprintf_s, snprintf, or vsnprintf",,CWE-120," sprintf(s, ""hello %s"", bug);",814237858ab012010f3355a49480dd6fa0a2cb8cf8356a98ac1c17c9febf6521
-test.c,21,2,4,buffer,sprintf,Does not check for buffer overflows (CWE-120),"Use sprintf_s, snprintf, or vsnprintf",,CWE-120," sprintf(s, gettext(""hello %s""), bug);",b793f18f143fb2297c49e0639384ad73db86eb01a44377aa4d5d09b44b03d747
-test.c,22,2,4,format,sprintf,Potential format string problem (CWE-134),Make format string constant,,CWE-134," sprintf(s, unknown, bug);",16ebc2ff96ee4bab2695783709e97b597ca9c8b8cc149e33aed859f0fafd3431
-test.c,23,2,4,format,printf,"If format strings can be influenced by an attacker, they can be exploited (CWE-134)",Use a constant for the format specification,,CWE-134," printf(bf, x);",46f42896019245d2dffc4caf4fe018b073ce2a58203676eaa28b6374558a5b5d
-test.c,25,2,4,buffer,scanf,"The scanf() family's %s operation, without a limit specification, permits buffer overflows (CWE-120, CWE-20)","Specify a limit to %s, or use a different input function",,"CWE-120, CWE-20"," scanf(""%s"", s);",3f169dd9fe508f70438f818770a3cb8b0f228e4245ea11a929a5fb0a7839fd5f
-test.c,27,2,4,buffer,scanf,"The scanf() family's %s operation, without a limit specification, permits buffer overflows (CWE-120, CWE-20)","Specify a limit to %s, or use a different input function",,"CWE-120, CWE-20"," scanf(""%s"", s);",3f169dd9fe508f70438f818770a3cb8b0f228e4245ea11a929a5fb0a7839fd5f
-test.c,38,2,4,format,syslog,"If syslog's format strings can be influenced by an attacker, they can be exploited (CWE-134)",Use a constant format string for syslog,,CWE-134," syslog(LOG_ERR, attacker_string);",22e98963d5af7b197a090bd522d2d39b8d8ee7bdf08453fd2008939c92cd9677
-test.c,49,3,4,buffer,_mbscpy,Does not check for buffer overflows when copying to destination [MS-banned] (CWE-120),Consider using a function version that stops copying at the end of the buffer,,CWE-120,"  _mbscpy(d,s); /* like strcpy, this doesn't check for buffer overflow */",e00a4a1a0a3603db98a23fcff3c9cdfd9012f5a81826814d9508e0f22089b993
-test.c,56,3,4,buffer,lstrcat,Does not check for buffer overflows when concatenating to destination [MS-banned] (CWE-120),,,CWE-120,"  lstrcat(d,s);",364b4c512862fdccbca27d2fa7737995b5d24b637a760976c940ae636218d340
-test.c,79,3,3,shell,CreateProcess,This causes a new process to execute and is difficult to use safely (CWE-78),"Specify the application path in the first argument, NOT as part of the second, or embedded spaces could allow an attacker to force a different program to run",,CWE-78,"  CreateProcess(NULL, ""C:\\Program Files\\GoodGuy\\GoodGuy.exe -x"", """");",3c712b38d0857bde3832d85ad35ac9859be55c5f5f1c20af659a577dd4d0acbf
-test.c,79,3,3,shell,CreateProcess,This causes a new process to execute and is difficult to use safely (CWE-78),"Specify the application path in the first argument, NOT as part of the second, or embedded spaces could allow an attacker to force a different program to run",,CWE-78,"  CreateProcess(NULL, ""C:\\Program Files\\GoodGuy\\GoodGuy.exe -x"", """");",3c712b38d0857bde3832d85ad35ac9859be55c5f5f1c20af659a577dd4d0acbf
-test.c,81,10,3,misc,LoadLibraryEx,"Ensure that the full path to the library is specified, or current directory may be used (CWE-829, CWE-20)",Use a flag like LOAD_LIBRARY_SEARCH_SYSTEM32 or LOAD_LIBRARY_SEARCH_APPLICATION_DIR to search only desired folders,,"CWE-829, CWE-20","  (void) LoadLibraryEx(L""user32.dll"", nullptr, LOAD_LIBRARY_AS_DATAFILE);",b1f99ecaa31e682487d795afbf03282fd56ad9f2aa630d0196219b277d2a68c9
-test.c,99,20,3,buffer,getopt_long,"Some older implementations do not protect against internal buffer overflows (CWE-120, CWE-20)","Check implementation on installation, or limit the size of all string inputs",,"CWE-120, CWE-20","    while ((optc = getopt_long (argc, argv, ""a"",longopts, NULL )) != EOF) {",5bedf6e5bccf596008ef191ec4c5d4cc51a32cff0c05ef62d5f10fab93d0cc24
-test.c,16,2,2,buffer,strcpy,Does not check for buffer overflows when copying to destination [MS-banned] (CWE-120),"Consider using snprintf, strcpy_s, or strlcpy (warning: strncpy easily misused)",Risk is low because the source is a constant string.,CWE-120," strcpy(a, gettext(""Hello there"")); // Did this work?",d64070fb93ff0bb797fb926f4dddc7212d42f77e288d5ceb0cd30ed2979fa28d
-test.c,19,2,2,buffer,sprintf,Does not check for buffer overflows (CWE-120),"Use sprintf_s, snprintf, or vsnprintf",Risk is low because the source has a constant maximum length.,CWE-120," sprintf(s, ""hello"");",907b46be1c3ea7b38f90a4d1b0f43b7751cd8cbe38fae840930ff006b702157d
-test.c,45,3,2,buffer,char,"Statically-sized arrays can be improperly restricted, leading to potential overflows or other issues (CWE-119!/CWE-120)","Perform bounds checking, use functions that limit length, or ensure that the size is larger than the maximum possible length",,CWE-119!/CWE-120,  char d[20];,36c87517700337a59cc3ad3218cfdde56cad37d69cdeccee5a55ab232d5c7946
-test.c,46,3,2,buffer,char,"Statically-sized arrays can be improperly restricted, leading to potential overflows or other issues (CWE-119!/CWE-120)","Perform bounds checking, use functions that limit length, or ensure that the size is larger than the maximum possible length",,CWE-119!/CWE-120,  char s[20];,213de8e8815fc84c423b55fd845fea541f25744718e486234364bb457863b597
-test.c,50,3,2,buffer,memcpy,Does not check for buffer overflows when copying to destination (CWE-120),Make sure destination can always hold the source data,,CWE-120,"  memcpy(d,s); // fail - no size",e667b352fb0748c67b607b11577b11bad87545779c39923e61839dd04056055f
-test.c,53,3,2,buffer,memcpy,Does not check for buffer overflows when copying to destination (CWE-120),Make sure destination can always hold the source data,,CWE-120,"  memcpy(&n,s,sizeof(s)); // fail - sizeof not of destination",01bcc2c8ba2d928ac3315b4dcc6593042ea05e62888a10a6d2cf16797a65ed32
-test.c,54,3,2,buffer,memcpy,Does not check for buffer overflows when copying to destination (CWE-120),Make sure destination can always hold the source data,,CWE-120,"  memcpy(d,s,n); // fail - size unguessable",2517a2fb5981193a6017cca660d16e85aab133706cbec302df97aaa623fc77ef
-test.c,55,3,2,buffer,CopyMemory,Does not check for buffer overflows when copying to destination (CWE-120),Make sure destination can always hold the source data,,CWE-120,"  CopyMemory(d,s);",977f8c805ddd76ff32e0f7aea08701ba97d9ce6955136e98b308ed4f70eb2e11
-test.c,105,7,2,misc,fopen,"Check when opening files - can an attacker redirect it (via symlinks), force the opening of special file type (e.g., device files), move things around to create a race condition, control its ancestors, or change its contents? (CWE-362)",,,CWE-362,"  f = fopen(""/etc/passwd"", ""r""); ",2ec6928c77a8b54caa61d0459f367c4394ee1f5e6f488753f587bfa9c780bad8
-test.c,15,2,1,buffer,strcpy,Does not check for buffer overflows when copying to destination [MS-banned] (CWE-120),"Consider using snprintf, strcpy_s, or strlcpy (warning: strncpy easily misused)",Risk is low because the source is a constant character.,CWE-120," strcpy(a, ""\n""); // Did this work?",0badc5f4c500d17b42794feaca54ee0f49e607a32510af3ed749579001017edb
-test.c,18,2,1,buffer,sprintf,Does not check for buffer overflows (CWE-120),"Use sprintf_s, snprintf, or vsnprintf",Risk is low because the source is a constant character.,CWE-120," sprintf(s, ""\n"");",c65fbd60851f3c8ace22332805966606488c0d242c1823493c582e267609b1a7
-test.c,26,2,1,buffer,scanf,It's unclear if the %s limit in the format string is small enough (CWE-120),"Check that the limit is sufficiently small, or use a different input function",,CWE-120," scanf(""%10s"", s);",e24c4c801f10acfa93098b2bef58524efe4f88237f2dd8b58be9afa838616afe
-test.c,57,3,1,buffer,strncpy,Easily used incorrectly; doesn't always \0-terminate or check for invalid pointers [MS-banned] (CWE-120),,,CWE-120,"  strncpy(d,s);",8fa14bf72393a00f667ffcc06b7b7e5f0b6d2f16d8d67444db06b0deb35b5f5e
-test.c,58,3,1,buffer,_tcsncpy,Easily used incorrectly; doesn't always \0-terminate or check for invalid pointers [MS-banned] (CWE-120),,,CWE-120,"  _tcsncpy(d,s);",691fabd4ca960a00e4c538eee0187ee0fdf59bd43dd71e792c14175150369b8b
-test.c,59,3,1,buffer,strncat,"Easily used incorrectly (e.g., incorrectly computing the correct maximum size to add) [MS-banned] (CWE-120)","Consider strcat_s, strlcat, snprintf, or automatically resizing strings",,CWE-120,"  strncat(d,s,10);",dd92f996a554bfbc038bea27640ba25dcf298383140a8330dca7cdacf493a701
-test.c,62,7,1,buffer,strlen,Does not handle strings that are not \0-terminated; if given one it may perform an over-read (it could cause a crash if unprotected) (CWE-126),,,CWE-126,  n = strlen(d);,db7201c7df7f543ea76febb060bda167e414e71e3d18095fe1def69f8c47a4f6
-test.c,68,3,1,buffer,MultiByteToWideChar,"Requires maximum length in CHARACTERS, not bytes (CWE-120)",,"Risk is very low, the length appears to be in characters not bytes.",CWE-120,"  MultiByteToWideChar(CP_ACP,0,szName,-1,wszUserName,sizeof(wszUserName)/sizeof(wszUserName[0]));",1813fc329227b38abae867d8023a9e29c7517d679fe55c86f8300dde681b6470
-test.c,70,3,1,buffer,MultiByteToWideChar,"Requires maximum length in CHARACTERS, not bytes (CWE-120)",,"Risk is very low, the length appears to be in characters not bytes.",CWE-120,"  MultiByteToWideChar(CP_ACP,0,szName,-1,wszUserName,sizeof wszUserName /sizeof(wszUserName[0]));",7c6cdcb10ad3a16b8bfd56e3dac84829f9bc3e39d4dde74a2be9bbe000102fc5
+File,Line,Column,DefaultLevel,Level,Category,Name,Warning,Suggestion,Note,CWEs,Context,Fingerprint,ToolVersion,RuleId,HelpUri
+test.c,32,2,5,5,buffer,gets,"Does not check for buffer overflows (CWE-120, CWE-20).",Use fgets() instead.,,"CWE-120, CWE-20", gets(f);,6a5bb383fb44030b0d9428b17359e94ba3979bc1ce702be450427f85592c649a,2.0.19,FF1014,https://cwe.mitre.org/data/definitions/120.html
+test.c,60,3,1,5,buffer,strncat,"Easily used incorrectly (e.g., incorrectly computing the correct maximum size to add) [MS-banned] (CWE-120).","Consider strcat_s, strlcat, snprintf, or automatically resizing strings.","Risk is high; the length parameter appears to be a constant, instead of computing the number of characters left.",CWE-120,"  strncat(d,s,sizeof(d)); /* Misuse - this should be flagged as riskier. */",cbd19c308547e79af13436d8f7dbcf6c62e49e4f62ba9aee38fbef29e0772f74,2.0.19,FF1010,https://cwe.mitre.org/data/definitions/120.html
+test.c,61,3,1,5,buffer,_tcsncat,"Easily used incorrectly (e.g., incorrectly computing the correct maximum size to add) [MS-banned] (CWE-120).","Consider strcat_s, strlcat, or automatically resizing strings.","Risk is high; the length parameter appears to be a constant, instead of computing the number of characters left.",CWE-120,"  _tcsncat(d,s,sizeof(d)); /* Misuse - flag as riskier */",c3f6ba2c710efc878e66df4578894fd408452cb7cdec7ae6f492a3b1796f8c42,2.0.19,FF1011,https://cwe.mitre.org/data/definitions/120.html
+test.c,64,3,2,5,buffer,MultiByteToWideChar,"Requires maximum length in CHARACTERS, not bytes (CWE-120).",,"Risk is high, it appears that the size is given as bytes, but the function requires size as characters.",CWE-120,"  MultiByteToWideChar(CP_ACP,0,szName,-1,wszUserName,sizeof(wszUserName));",4f5b73ff337a54d6e1d9a369659ca0ddb4f80e6b7e38a17e5b112f6d3e266e69,2.0.19,FF1023,https://cwe.mitre.org/data/definitions/120.html
+test.c,66,3,2,5,buffer,MultiByteToWideChar,"Requires maximum length in CHARACTERS, not bytes (CWE-120).",,"Risk is high, it appears that the size is given as bytes, but the function requires size as characters.",CWE-120,"  MultiByteToWideChar(CP_ACP,0,szName,-1,wszUserName,sizeof wszUserName);",9ecdc1e903acc16a646bf7909a630ae22a7593b70952c39ce6bd9c5a23fad0fd,2.0.19,FF1023,https://cwe.mitre.org/data/definitions/120.html
+test.c,77,3,5,5,misc,SetSecurityDescriptorDacl,"Never create NULL ACLs; an attacker can set it to Everyone (Deny All Access), which would even forbid administrator access (CWE-732).",,,CWE-732,"  SetSecurityDescriptorDacl(&sd,TRUE,NULL,FALSE);",5fed1e135b593b4c943e66e89a26ff131eba18b83a32a8af37d1c0bd7b01aadb,2.0.19,FF1060,https://cwe.mitre.org/data/definitions/732.html
+test.c,77,3,5,5,misc,SetSecurityDescriptorDacl,"Never create NULL ACLs; an attacker can set it to Everyone (Deny All Access), which would even forbid administrator access (CWE-732).",,,CWE-732,"  SetSecurityDescriptorDacl(&sd,TRUE,NULL,FALSE);",5fed1e135b593b4c943e66e89a26ff131eba18b83a32a8af37d1c0bd7b01aadb,2.0.19,FF1060,https://cwe.mitre.org/data/definitions/732.html
+test.c,17,2,4,4,buffer,strcpy,Does not check for buffer overflows when copying to destination [MS-banned] (CWE-120).,"Consider using snprintf, strcpy_s, or strlcpy (warning: strncpy easily misused).",,CWE-120," strcpy(b, a);",c01c8472bb53022e912da4da2faebc67d537855da324020c44bfd5e608a79b77,2.0.19,FF1001,https://cwe.mitre.org/data/definitions/120.html
+test.c,20,2,4,4,buffer,sprintf,Does not check for buffer overflows (CWE-120).,"Use sprintf_s, snprintf, or vsnprintf.",,CWE-120," sprintf(s, ""hello %s"", bug);",814237858ab012010f3355a49480dd6fa0a2cb8cf8356a98ac1c17c9febf6521,2.0.19,FF1015,https://cwe.mitre.org/data/definitions/120.html
+test.c,21,2,4,4,buffer,sprintf,Does not check for buffer overflows (CWE-120).,"Use sprintf_s, snprintf, or vsnprintf.",,CWE-120," sprintf(s, gettext(""hello %s""), bug);",b793f18f143fb2297c49e0639384ad73db86eb01a44377aa4d5d09b44b03d747,2.0.19,FF1015,https://cwe.mitre.org/data/definitions/120.html
+test.c,22,2,4,4,format,sprintf,Potential format string problem (CWE-134).,Make format string constant.,,CWE-134," sprintf(s, unknown, bug);",16ebc2ff96ee4bab2695783709e97b597ca9c8b8cc149e33aed859f0fafd3431,2.0.19,FF1015,https://cwe.mitre.org/data/definitions/134.html
+test.c,23,2,4,4,format,printf,"If format strings can be influenced by an attacker, they can be exploited (CWE-134).",Use a constant for the format specification.,,CWE-134," printf(bf, x);",46f42896019245d2dffc4caf4fe018b073ce2a58203676eaa28b6374558a5b5d,2.0.19,FF1016,https://cwe.mitre.org/data/definitions/134.html
+test.c,25,2,4,4,buffer,scanf,"The scanf() family's %s operation, without a limit specification, permits buffer overflows (CWE-120, CWE-20).","Specify a limit to %s, or use a different input function.",,"CWE-120, CWE-20"," scanf(""%s"", s);",3f169dd9fe508f70438f818770a3cb8b0f228e4245ea11a929a5fb0a7839fd5f,2.0.19,FF1020,https://cwe.mitre.org/data/definitions/120.html
+test.c,27,2,4,4,buffer,scanf,"The scanf() family's %s operation, without a limit specification, permits buffer overflows (CWE-120, CWE-20).","Specify a limit to %s, or use a different input function.",,"CWE-120, CWE-20"," scanf(""%s"", s);",3f169dd9fe508f70438f818770a3cb8b0f228e4245ea11a929a5fb0a7839fd5f,2.0.19,FF1020,https://cwe.mitre.org/data/definitions/120.html
+test.c,38,2,4,4,format,syslog,"If syslog's format strings can be influenced by an attacker, they can be exploited (CWE-134).",Use a constant format string for syslog.,,CWE-134," syslog(LOG_ERR, attacker_string);",22e98963d5af7b197a090bd522d2d39b8d8ee7bdf08453fd2008939c92cd9677,2.0.19,FF1018,https://cwe.mitre.org/data/definitions/134.html
+test.c,49,3,4,4,buffer,_mbscpy,Does not check for buffer overflows when copying to destination [MS-banned] (CWE-120).,Consider using a function version that stops copying at the end of the buffer.,,CWE-120,"  _mbscpy(d,s); /* like strcpy, this doesn't check for buffer overflow */",e00a4a1a0a3603db98a23fcff3c9cdfd9012f5a81826814d9508e0f22089b993,2.0.19,FF1003,https://cwe.mitre.org/data/definitions/120.html
+test.c,56,3,4,4,buffer,lstrcat,Does not check for buffer overflows when concatenating to destination [MS-banned] (CWE-120).,,,CWE-120,"  lstrcat(d,s);",364b4c512862fdccbca27d2fa7737995b5d24b637a760976c940ae636218d340,2.0.19,FF1006,https://cwe.mitre.org/data/definitions/120.html
+test.c,79,3,3,3,shell,CreateProcess,This causes a new process to execute and is difficult to use safely (CWE-78).,"Specify the application path in the first argument, NOT as part of the second, or embedded spaces could allow an attacker to force a different program to run.",,CWE-78,"  CreateProcess(NULL, ""C:\\Program Files\\GoodGuy\\GoodGuy.exe -x"", """");",3c712b38d0857bde3832d85ad35ac9859be55c5f5f1c20af659a577dd4d0acbf,2.0.19,FF1046,https://cwe.mitre.org/data/definitions/78.html
+test.c,79,3,3,3,shell,CreateProcess,This causes a new process to execute and is difficult to use safely (CWE-78).,"Specify the application path in the first argument, NOT as part of the second, or embedded spaces could allow an attacker to force a different program to run.",,CWE-78,"  CreateProcess(NULL, ""C:\\Program Files\\GoodGuy\\GoodGuy.exe -x"", """");",3c712b38d0857bde3832d85ad35ac9859be55c5f5f1c20af659a577dd4d0acbf,2.0.19,FF1046,https://cwe.mitre.org/data/definitions/78.html
+test.c,81,10,3,3,misc,LoadLibraryEx,"Ensure that the full path to the library is specified, or current directory may be used (CWE-829, CWE-20).",Use a flag like LOAD_LIBRARY_SEARCH_SYSTEM32 or LOAD_LIBRARY_SEARCH_APPLICATION_DIR to search only desired folders.,,"CWE-829, CWE-20","  (void) LoadLibraryEx(L""user32.dll"", nullptr, LOAD_LIBRARY_AS_DATAFILE);",b1f99ecaa31e682487d795afbf03282fd56ad9f2aa630d0196219b277d2a68c9,2.0.19,FF1059,https://cwe.mitre.org/data/definitions/829.html
+test.c,99,20,3,3,buffer,getopt_long,"Some older implementations do not protect against internal buffer overflows (CWE-120, CWE-20).","Check implementation on installation, or limit the size of all string inputs.",,"CWE-120, CWE-20","    while ((optc = getopt_long (argc, argv, ""a"",longopts, NULL )) != EOF) {",5bedf6e5bccf596008ef191ec4c5d4cc51a32cff0c05ef62d5f10fab93d0cc24,2.0.19,FF1027,https://cwe.mitre.org/data/definitions/120.html
+test.c,16,2,4,2,buffer,strcpy,Does not check for buffer overflows when copying to destination [MS-banned] (CWE-120).,"Consider using snprintf, strcpy_s, or strlcpy (warning: strncpy easily misused).",Risk is low because the source is a constant string.,CWE-120," strcpy(a, gettext(""Hello there"")); // Did this work?",d64070fb93ff0bb797fb926f4dddc7212d42f77e288d5ceb0cd30ed2979fa28d,2.0.19,FF1001,https://cwe.mitre.org/data/definitions/120.html
+test.c,19,2,4,2,buffer,sprintf,Does not check for buffer overflows (CWE-120).,"Use sprintf_s, snprintf, or vsnprintf.",Risk is low because the source has a constant maximum length.,CWE-120," sprintf(s, ""hello"");",907b46be1c3ea7b38f90a4d1b0f43b7751cd8cbe38fae840930ff006b702157d,2.0.19,FF1015,https://cwe.mitre.org/data/definitions/120.html
+test.c,45,3,2,2,buffer,char,"Statically-sized arrays can be improperly restricted, leading to potential overflows or other issues (CWE-119!/CWE-120).","Perform bounds checking, use functions that limit length, or ensure that the size is larger than the maximum possible length.",,CWE-119!/CWE-120,  char d[20];,36c87517700337a59cc3ad3218cfdde56cad37d69cdeccee5a55ab232d5c7946,2.0.19,FF1013,https://cwe.mitre.org/data/definitions/119.html
+test.c,46,3,2,2,buffer,char,"Statically-sized arrays can be improperly restricted, leading to potential overflows or other issues (CWE-119!/CWE-120).","Perform bounds checking, use functions that limit length, or ensure that the size is larger than the maximum possible length.",,CWE-119!/CWE-120,  char s[20];,213de8e8815fc84c423b55fd845fea541f25744718e486234364bb457863b597,2.0.19,FF1013,https://cwe.mitre.org/data/definitions/119.html
+test.c,50,3,2,2,buffer,memcpy,Does not check for buffer overflows when copying to destination (CWE-120).,Make sure destination can always hold the source data.,,CWE-120,"  memcpy(d,s); // fail - no size",e667b352fb0748c67b607b11577b11bad87545779c39923e61839dd04056055f,2.0.19,FF1004,https://cwe.mitre.org/data/definitions/120.html
+test.c,53,3,2,2,buffer,memcpy,Does not check for buffer overflows when copying to destination (CWE-120).,Make sure destination can always hold the source data.,,CWE-120,"  memcpy(&n,s,sizeof(s)); // fail - sizeof not of destination",01bcc2c8ba2d928ac3315b4dcc6593042ea05e62888a10a6d2cf16797a65ed32,2.0.19,FF1004,https://cwe.mitre.org/data/definitions/120.html
+test.c,54,3,2,2,buffer,memcpy,Does not check for buffer overflows when copying to destination (CWE-120).,Make sure destination can always hold the source data.,,CWE-120,"  memcpy(d,s,n); // fail - size unguessable",2517a2fb5981193a6017cca660d16e85aab133706cbec302df97aaa623fc77ef,2.0.19,FF1004,https://cwe.mitre.org/data/definitions/120.html
+test.c,55,3,2,2,buffer,CopyMemory,Does not check for buffer overflows when copying to destination (CWE-120).,Make sure destination can always hold the source data.,,CWE-120,"  CopyMemory(d,s);",977f8c805ddd76ff32e0f7aea08701ba97d9ce6955136e98b308ed4f70eb2e11,2.0.19,FF1004,https://cwe.mitre.org/data/definitions/120.html
+test.c,105,7,2,2,misc,fopen,"Check when opening files - can an attacker redirect it (via symlinks), force the opening of special file type (e.g., device files), move things around to create a race condition, control its ancestors, or change its contents? (CWE-362).",,,CWE-362,"  f = fopen(""/etc/passwd"", ""r""); ",2ec6928c77a8b54caa61d0459f367c4394ee1f5e6f488753f587bfa9c780bad8,2.0.19,FF1040,https://cwe.mitre.org/data/definitions/362.html
+test.c,15,2,4,1,buffer,strcpy,Does not check for buffer overflows when copying to destination [MS-banned] (CWE-120).,"Consider using snprintf, strcpy_s, or strlcpy (warning: strncpy easily misused).",Risk is low because the source is a constant character.,CWE-120," strcpy(a, ""\n""); // Did this work?",0badc5f4c500d17b42794feaca54ee0f49e607a32510af3ed749579001017edb,2.0.19,FF1001,https://cwe.mitre.org/data/definitions/120.html
+test.c,18,2,4,1,buffer,sprintf,Does not check for buffer overflows (CWE-120).,"Use sprintf_s, snprintf, or vsnprintf.",Risk is low because the source is a constant character.,CWE-120," sprintf(s, ""\n"");",c65fbd60851f3c8ace22332805966606488c0d242c1823493c582e267609b1a7,2.0.19,FF1015,https://cwe.mitre.org/data/definitions/120.html
+test.c,26,2,4,1,buffer,scanf,It's unclear if the %s limit in the format string is small enough (CWE-120).,"Check that the limit is sufficiently small, or use a different input function.",,CWE-120," scanf(""%10s"", s);",e24c4c801f10acfa93098b2bef58524efe4f88237f2dd8b58be9afa838616afe,2.0.19,FF1020,https://cwe.mitre.org/data/definitions/120.html
+test.c,57,3,1,1,buffer,strncpy,Easily used incorrectly; doesn't always \0-terminate or check for invalid pointers [MS-banned] (CWE-120).,,,CWE-120,"  strncpy(d,s);",8fa14bf72393a00f667ffcc06b7b7e5f0b6d2f16d8d67444db06b0deb35b5f5e,2.0.19,FF1008,https://cwe.mitre.org/data/definitions/120.html
+test.c,58,3,1,1,buffer,_tcsncpy,Easily used incorrectly; doesn't always \0-terminate or check for invalid pointers [MS-banned] (CWE-120).,,,CWE-120,"  _tcsncpy(d,s);",691fabd4ca960a00e4c538eee0187ee0fdf59bd43dd71e792c14175150369b8b,2.0.19,FF1009,https://cwe.mitre.org/data/definitions/120.html
+test.c,59,3,1,1,buffer,strncat,"Easily used incorrectly (e.g., incorrectly computing the correct maximum size to add) [MS-banned] (CWE-120).","Consider strcat_s, strlcat, snprintf, or automatically resizing strings.",,CWE-120,"  strncat(d,s,10);",dd92f996a554bfbc038bea27640ba25dcf298383140a8330dca7cdacf493a701,2.0.19,FF1010,https://cwe.mitre.org/data/definitions/120.html
+test.c,62,7,1,1,buffer,strlen,Does not handle strings that are not \0-terminated; if given one it may perform an over-read (it could cause a crash if unprotected) (CWE-126).,,,CWE-126,  n = strlen(d);,db7201c7df7f543ea76febb060bda167e414e71e3d18095fe1def69f8c47a4f6,2.0.19,FF1022,https://cwe.mitre.org/data/definitions/126.html
+test.c,68,3,2,1,buffer,MultiByteToWideChar,"Requires maximum length in CHARACTERS, not bytes (CWE-120).",,"Risk is very low, the length appears to be in characters not bytes.",CWE-120,"  MultiByteToWideChar(CP_ACP,0,szName,-1,wszUserName,sizeof(wszUserName)/sizeof(wszUserName[0]));",1813fc329227b38abae867d8023a9e29c7517d679fe55c86f8300dde681b6470,2.0.19,FF1023,https://cwe.mitre.org/data/definitions/120.html
+test.c,70,3,2,1,buffer,MultiByteToWideChar,"Requires maximum length in CHARACTERS, not bytes (CWE-120).",,"Risk is very low, the length appears to be in characters not bytes.",CWE-120,"  MultiByteToWideChar(CP_ACP,0,szName,-1,wszUserName,sizeof wszUserName /sizeof(wszUserName[0]));",7c6cdcb10ad3a16b8bfd56e3dac84829f9bc3e39d4dde74a2be9bbe000102fc5,2.0.19,FF1023,https://cwe.mitre.org/data/definitions/120.html

test/correct-results.html

@@ -9,7 +9,7 @@
 <body>
 <h1>Flawfinder Results</h1>
 Here are the security scan results from
-<a href="https://dwheeler.com/flawfinder">Flawfinder version 2.0.15</a>,
+<a href="https://dwheeler.com/flawfinder">Flawfinder version 2.0.19</a>,
 (C) 2001-2019 <a href="https://dwheeler.com">David A. Wheeler</a>.
 Number of rules (primarily dangerous function names) in C/C++ ruleset: 222
 <p>

test/correct-results.txt

@@ -1,4 +1,4 @@
-Flawfinder version 2.0.15, (C) 2001-2019 David A. Wheeler.
+Flawfinder version 2.0.19, (C) 2001-2019 David A. Wheeler.
 Number of rules (primarily dangerous function names) in C/C++ ruleset: 222
 Examining test.c
 Examining test2.c

test/makefile

@@ -5,7 +5,7 @@
 PYTHON=python
 PYTHON2=python2
 PYTHON3=python3
-FLAWFINDER=../flawfinder
+FLAWFINDER=../flawfinder.py
 SETUPPY=../setup.py
 
 test_001: $(FLAWFINDER) test.c test2.c
@@ -16,29 +16,29 @@ test_001: $(FLAWFINDER) test.c test2.c
 	@echo "Testing for no ending newline:" >> test-results.txt
 	@$(PYTHON) $(FLAWFINDER) --omittime no-ending-newline.c | \
 	  grep 'Lines analyzed' >> test-results.txt
-	@diff -u correct-results.txt test-results.txt
+	@diff -u --ignore-all-space correct-results.txt test-results.txt
 
 test_002: $(FLAWFINDER) test.c test2.c
 	@echo 'test_002 (HTML output)'
 	@$(PYTHON) $(FLAWFINDER) --omittime --html --context test.c test2.c > test-results.html
-	@diff -u correct-results.html test-results.html
+	@diff -u --ignore-all-space correct-results.html test-results.html
 
 test_003: $(FLAWFINDER) test.c test2.c
 	@echo 'test_003 (CSV output)'
 	@$(PYTHON) $(FLAWFINDER) --csv test.c test2.c > test-results.csv
-	@diff -u correct-results.csv test-results.csv
+	@diff -u --ignore-all-space correct-results.csv test-results.csv
 
 test_004: $(FLAWFINDER) test.c
 	@echo 'test_004 (single-line)'
 	@$(PYTHON) $(FLAWFINDER) -m 5 -S -DC --quiet test.c > \
 	  test-results-004.txt
-	@diff -u correct-results-004.txt test-results-004.txt
+	@diff -u --ignore-all-space correct-results-004.txt test-results-004.txt
 
 test_005: $(FLAWFINDER) test-diff-005.patch test-patched.c
 	@echo 'test_005 (diff)'
 	@$(PYTHON) $(FLAWFINDER) -SQDC -P test-diff-005.patch \
 	  test-patched.c > test-results-005.txt
-	@diff -u correct-results-005.txt test-results-005.txt
+	@diff -u --ignore-all-space correct-results-005.txt test-results-005.txt
 
 test_006: $(FLAWFINDER) test.c
 	@echo 'test_006 (save/load hitlist)'
@@ -48,7 +48,7 @@ test_006: $(FLAWFINDER) test.c
 	@$(PYTHON) $(FLAWFINDER) -SQDC -m 5 \
 	  --loadhitlist test-saved-hitlist-006.txt > \
 	  test-results-006.txt
-	@diff -u correct-results-006.txt test-results-006.txt
+	@diff -u --ignore-all-space correct-results-006.txt test-results-006.txt
 
 test_007: $(SETUPPY)
 	@echo 'test_007 (setup.py sane)'
@@ -64,7 +64,7 @@ test_008: $(FLAWFINDER) test.c
 	@$(PYTHON) $(FLAWFINDER) -S -C --quiet --omittime \
 	  --diffhitlist  test-saved-hitlist-008.txt test.c > \
 	  test-results-008.txt
-	@diff -u correct-results-008.txt test-results-008.txt
+	@diff -u --ignore-all-space correct-results-008.txt test-results-008.txt
 
 test_009: $(FLAWFINDER) test-cpp-digit-separator.cpp
 	@echo 'test_009 (C++ digit separator)'
@@ -107,3 +107,4 @@ test-is-correct: test-results.txt
 	cp -p test-results-008.txt correct-results-008.txt
 
 .PHONY: test check test-is-correct
+