# Copyright (c) 2019 - 2026, Ilan Schnell; All Rights Reserved
# bitarray is published under the PSF license.
#
# Author: Ilan Schnell
"""
Tests for bitarray.util module
"""
import os
import sys
import math
import array
import base64
import binascii
import operator
import struct
import tempfile
import unittest
from io import StringIO
from functools import reduce
from random import (choice, choices, getrandbits, randrange, randint, random,
                    sample, seed)
from string import hexdigits, whitespace
from collections import Counter

from bitarray import (bitarray, frozenbitarray, decodetree, bits2bytes,
                      get_default_endian)
from bitarray.test_bitarray import Util, is_pypy, urandom_2, ENDIANS, PTRSIZE

from bitarray.util import (
    zeros, ones, urandom, random_k, random_p, pprint, strip, count_n,
    parity, gen_primes, sum_indices, xor_indices,
    count_and, count_or, count_xor, any_and, subset,
    correspond_all, byteswap, intervals,
    serialize, deserialize, ba2hex, hex2ba, ba2base, base2ba,
    ba2int, int2ba,
    sc_encode, sc_decode, sc_stat, rl_encode, rl_decode, vl_encode, vl_decode,
    huffman_code, canonical_huffman, canonical_decode,
)
from bitarray._util import uleb128_encode  # type: ignore


OPT_ENDIANS = ENDIANS + (None,)

# ---------------------------  zeros()  ones()  -----------------------------

class ZerosOnesTests(unittest.TestCase):

    def test_basic(self):
        for _ in range(50):
            a = choice([zeros(0), zeros(0, None), zeros(0, endian=None),
                        ones(0), ones(0, None), ones(0, endian=None)])
            self.assertEqual(a, bitarray())
            self.assertEqual(a.endian, get_default_endian())
            self.assertIs(type(a), bitarray)

            endian = choice(OPT_ENDIANS)
            n = randrange(100)

            a = choice([zeros(n, endian), zeros(n, endian=endian)])
            self.assertEqual(a.to01(), n * "0")
            self.assertEqual(a.endian, endian or get_default_endian())

            b = choice([ones(n, endian), ones(n, endian=endian)])
            self.assertEqual(b.to01(), n * "1")
            self.assertEqual(b.endian, endian or get_default_endian())

    def test_errors(self):
        for f in zeros, ones:
            self.assertRaises(TypeError, f) # no argument
            self.assertRaises(TypeError, f, '')
            self.assertRaises(TypeError, f, bitarray())
            self.assertRaises(TypeError, f, [])
            self.assertRaises(TypeError, f, 1.0)
            self.assertRaises(ValueError, f, -1)

            # endian not string
            for x in 0, 1, {}, [], False, True:
                self.assertRaises(TypeError, f, 0, x)
            # endian wrong string
            self.assertRaises(ValueError, f, 0, 'foo')

# -----------------------------  urandom()  ---------------------------------

class URandomTests(unittest.TestCase):

    def test_basic(self):
        for _ in range(20):
            a = choice([urandom(0), urandom(0, endian=None)])
            self.assertEqual(a, bitarray())
            self.assertEqual(a.endian, get_default_endian())

            endian = choice(OPT_ENDIANS)
            n = randrange(100)

            a = choice([urandom(n, endian), urandom(n, endian=endian)])
            self.assertEqual(len(a), n)
            self.assertEqual(a.endian, endian or get_default_endian())
            self.assertIs(type(a), bitarray)

    def test_errors(self):
        U = urandom
        self.assertRaises(TypeError, U)
        self.assertRaises(TypeError, U, '')
        self.assertRaises(TypeError, U, bitarray())
        self.assertRaises(TypeError, U, [])
        self.assertRaises(TypeError, U, 1.0)
        self.assertRaises(ValueError, U, -1)
        self.assertRaises(TypeError, U, 0, 1)
        self.assertRaises(ValueError, U, 0, 'foo')

    def test_count(self):
        a = urandom(10_000_000)
        # see if population is within expectation
        self.assertTrue(abs(a.count() - 5_000_000) <= 15_811)

# ----------------------------  random_k()  ---------------------------------

class Random_K_Tests(unittest.TestCase):

    def test_basic(self):
        for _ in range(250):
            endian = choice(OPT_ENDIANS)
            n = randrange(120)
            k = randint(0, n)
            a = random_k(n, k, endian)
            self.assertIs(type(a), bitarray)
            self.assertEqual(len(a), n)
            self.assertEqual(a.count(), k)
            self.assertEqual(a.endian, endian or get_default_endian())

    def test_inputs_and_edge_cases(self):
        R = random_k
        self.assertRaises(TypeError, R)
        self.assertRaises(TypeError, R, 4)
        self.assertRaises(TypeError, R, 1, "0.5")
        self.assertRaises(TypeError, R, 10, 4.0)
        self.assertRaises(TypeError, R, 10.0, 4)
        self.assertRaises(TypeError, R, 1, p=1)
        self.assertRaises(TypeError, R, 11, 5.5)  # see issue #239
        self.assertRaises(ValueError, R, -1, 0)
        for k in -1, 11:  # k is not 0 <= k <= n
            self.assertRaises(ValueError, R, 10, k)
        self.assertRaises(ValueError, R, 10, 7, 'foo')
        self.assertRaises(ValueError, R, 10, 7, endian='foo')
        for n in range(20):
            self.assertEqual(R(n, k=0), zeros(n))
            self.assertEqual(R(n, k=n), ones(n))

    def test_count(self):
        for n in range(10):  # test explicitly for small n
            for k in range(n + 1):
                a = random_k(n, k)
                self.assertEqual(len(a), n)
                self.assertEqual(a.count(), k)

        for _ in range(100):
            n = randrange(10_000)
            k = randint(0, n)
            a = random_k(n, k)
            self.assertEqual(len(a), n)
            self.assertEqual(a.count(), k)

    def test_active_bits(self):
        # test if all bits are active
        n = 240
        cum = zeros(n)
        for _ in range(1000):
            k = randint(30, 40)
            a = random_k(n, k)
            self.assertEqual(a.count(), k)
            cum |= a
            if cum.all():
                break
        else:
            self.fail()

    @unittest.skipIf(sys.version_info[:2] < (3, 8), "math.comb added in 3.8")
    def test_combinations(self):
        # for entire range of 0 <= k <= n, validate that random_k()
        # generates all possible combinations
        n = 7
        total = 0
        for k in range(n + 1):
            expected = math.comb(n, k)
            combs = set()
            for _ in range(10_000):
                combs.add(frozenbitarray(random_k(n, k)))
                if len(combs) == expected:
                    total += expected
                    break
            else:
                self.fail()
        self.assertEqual(total, 2 ** n)

    def collect_code_branches(self):
        # return list of bitarrays from all code branches of random_k()
        res = []
        # test small k (no .combine_half())
        res.append(random_k(300, 10))
        # general cases
        for k in 100, 500, 2_500, 4_000:
            res.append(random_k(5_000, k))
        return res

    def test_seed(self):
        # We ensure that after setting a seed value, random_k() will
        # always return the same random bitarrays.  However, we do not ensure
        # that these results will not change in future versions of bitarray.
        a = []
        for val in 654321, 654322, 654321, 654322:
            seed(val)
            a.append(self.collect_code_branches())
        self.assertEqual(a[0], a[2])
        self.assertEqual(a[1], a[3])
        for item0, item1 in zip(a[0], a[1]):
            self.assertNotEqual(item0, item1)
        # initialize seed with current system time again
        seed()


# ----------------------------  random_p()  ---------------------------------

HAVE_BINOMIALVARIATE = sys.version_info[:2] >= (3, 12)

@unittest.skipIf(HAVE_BINOMIALVARIATE, "test for Python <3.12 - "
                 "random.binomialvariate() missing")
class Random_P_Not_Implemented(unittest.TestCase):

    def test_not_implemented(self):
        self.assertRaises(NotImplementedError, random_p, 100, 0.25)


@unittest.skipIf(not HAVE_BINOMIALVARIATE, "Python 3.12+ required")
class Random_P_Tests(unittest.TestCase):

    def test_basic(self):
        for _ in range(250):
            endian = choice(OPT_ENDIANS)
            n = randrange(120)
            p = choice([0.0, 0.0001, 0.2, 0.5, 0.9, 1.0])
            a = random_p(n, p, endian)
            self.assertIs(type(a), bitarray)
            self.assertEqual(len(a), n)
            self.assertEqual(a.endian, endian or get_default_endian())

    def test_inputs_and_edge_cases(self):
        R = random_p
        self.assertRaises(TypeError, R)
        self.assertRaises(TypeError, R, 0.25)
        self.assertRaises(TypeError, R, 1, "0.5")
        self.assertRaises(ValueError, R, -1)
        self.assertRaises(ValueError, R, 1, -0.5)
        self.assertRaises(ValueError, R, 1, p=1.5)
        self.assertRaises(ValueError, R, 1, 0.15, 'foo')
        self.assertRaises(ValueError, R, 10, 0.5, endian='foo')
        self.assertEqual(R(0), bitarray())
        for n in range(30):
            self.assertEqual(R(n, 0), zeros(n))
            self.assertEqual(len(R(n, 0.5)), n)
            self.assertEqual(R(n, p=1), ones(n))
            self.assertRaises(ValueError, R, n, math.nan)

    def test_default(self):
        a = random_p(10_000_000)  # p defaults to 0.5
        # see if population is within expectation
        self.assertTrue(abs(a.count() - 5_000_000) <= 15_811)

    def test_count(self):
        for _ in range(500):
            n = choice([randrange(4, 120), randrange(100, 1000)])
            p = choice([0.0001, 0.001, 0.01, 0.1, 0.25, 0.5, 0.9])
            sigma = math.sqrt(n * p * (1.0 - p))
            a = random_p(n, p)
            self.assertEqual(len(a), n)
            self.assertTrue(abs(a.count() - n * p) < max(4, 10 * sigma))

    def collect_code_branches(self):
        # return list of bitarrays from all code branches of random_p()
        res = []
        # for default p=0.5, random_p uses getrandbits
        res.append(random_p(32))
        # test small p
        res.append(random_p(5_000, 0.002))
        # small n (note that p=0.4 will call the "literal definition" case)
        res.append(random_p(15, 0.4))
        # general cases
        for p in 0.1, 0.2, 0.375, 0.4999, 0.7:
            res.append(random_p(150, p))
        return res

    def test_seed(self):
        # We ensure that after setting a seed value, random_p() will always
        # return the same random bitarrays.  However, we do not ensure that
        # these results will not change in future versions of bitarray.
        a = []
        for val in 123456, 123457, 123456, 123457:
            seed(val)
            a.append(self.collect_code_branches())
        self.assertEqual(a[0], a[2])
        self.assertEqual(a[1], a[3])
        for item0, item1 in zip(a[0], a[1]):
            self.assertNotEqual(item0, item1)
        # initialize seed with current system time again
        seed()


# ----------------------------  gen_primes()  -------------------------------

class PrimeTests(unittest.TestCase):

    primes = [
        2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61,
        67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127, 131, 137,
        139, 149, 151, 157, 163, 167, 173, 179, 181, 191, 193, 197, 199, 211,
        223, 227, 229, 233, 239, 241, 251, 257, 263, 269, 271, 277, 281, 283,
        293, 307, 311, 313, 317, 331, 337, 347, 349, 353, 359, 367, 373, 379,
        383, 389, 397, 401, 409, 419, 421, 431, 433, 439, 443, 449, 457, 461,
    ]

    def test_errors(self):
        P = gen_primes
        self.assertRaises(TypeError, P, 3, 1)
        self.assertRaises(TypeError, P, "1")
        self.assertRaises(TypeError, P, "1.0")
        self.assertRaises(TypeError, P, 123.0)
        self.assertRaises(ValueError, P, -1)
        self.assertRaises(TypeError, P, 8, 4)
        self.assertRaises(TypeError, P, 8, foo="big")
        self.assertRaises(ValueError, P, 8, "foo")
        self.assertRaises(ValueError, P, 8, endian="foo")

    def test_explicit(self):
        for n in range(230):
            endian = choice(OPT_ENDIANS)
            odd = getrandbits(1)
            a = gen_primes(n, endian, odd)
            self.assertEqual(len(a), n)
            self.assertEqual(a.endian, endian or get_default_endian())
            if odd:
                lst = [2] + [2 * i + 1 for i in a.search(1)]
            else:
                lst = [i for i in a.search(1)]
            self.assertEqual(lst, self.primes[:len(lst)])

    def test_cmp(self):
        N = 10_000
        c = ones(N)
        c[:2] = 0
        for i in range(int(math.sqrt(N) + 1.0)):
            if c[i]:
                c[i * i :: i] = 0
        self.assertEqual(list(c.search(1, 0, 462)), self.primes)

        for _ in range(20):
            n = randrange(N)
            endian = choice(ENDIANS)
            a = gen_primes(n, endian=endian)
            self.assertEqual(a, c[:n])
            self.assertEqual(a.endian, endian)

            b = gen_primes(n // 2, endian, odd=True)
            self.assertEqual(b, a[1::2])
            self.assertEqual(b, c[1:n:2])

        for _ in range(20):
            i = randrange(10, 100)
            x = randint(-1, 1)
            n = i * i + x
            self.assertEqual(gen_primes(n), c[:n])
            self.assertEqual(gen_primes(n // 2, odd=1), c[1:n:2])

        self.assertEqual(gen_primes(N), c)
        self.assertEqual(gen_primes(N // 2, odd=1), c[1::2])

    def test_count(self):
        for n, count, sum_p, sum_sqr_p in [
                (    10,    4,        17,             87),
                (   100,   25,     1_060,         65_796),
                ( 1_000,  168,    76_127,     49_345_379),
                (10_000, 1229, 5_736_396, 37_546_387_960),
        ]:
            a = gen_primes(n)
            self.assertEqual(len(a), n)
            self.assertEqual(a.count(), count)
            self.assertEqual(sum_indices(a), sum_p)
            self.assertEqual(sum_indices(a, 2), sum_sqr_p)
            b = gen_primes(n // 2, odd=1)
            self.assertEqual(len(b), n // 2)
            self.assertEqual(b.count() + 1, count)  # +1 because of prime 2
            self.assertEqual(b, a[1::2])

# -----------------------------  pprint()  ----------------------------------

class PPrintTests(unittest.TestCase):

    def test_example(self):
        a = gen_primes(100)
        f = StringIO()
        pprint(a, f, indent=3, group=10, width=60)
        self.assertEqual(
            f.getvalue(), "bitarray('''\n"
            "   0011010100 0101000101 0001000001 0100000100 0101000100\n"
            "   0001000001 0100000100 0101000001 0001000001 0000000100\n"
            "''')\n")

    @staticmethod
    def get_code_string(a):
        f = StringIO()
        pprint(a, stream=f)
        return f.getvalue()

    def round_trip(self, a):
        b = eval(self.get_code_string(a))
        self.assertEqual(b, a)
        self.assertIs(type(b), type(a))

    def test_bitarray(self):
        a = bitarray('110')
        self.assertEqual(self.get_code_string(a), "bitarray('110')\n")
        self.round_trip(a)

    def test_frozenbitarray(self):
        a = frozenbitarray('01')
        self.assertEqual(self.get_code_string(a), "frozenbitarray('01')\n")
        self.round_trip(a)

    def test_formatting(self):
        a = bitarray(200)
        for width in range(40, 130, 10):
            for n in range(1, 10):
                f = StringIO()
                pprint(a, stream=f, group=n, width=width)
                r = f.getvalue()
                self.assertEqual(eval(r), a)
                s = r.strip("bitary(')\n")
                for group in s.split()[:-1]:
                    self.assertEqual(len(group), n)
                for line in s.split('\n'):
                    self.assertTrue(len(line) < width)

    def test_narrow_width(self):
        a = bitarray('101')
        f = StringIO()
        pprint(a, stream=f, width=6)
        self.assertEqual(eval(f.getvalue()), a)

    def test_fallback(self):
        for a in None, 'asd', [1, 2], bitarray(), frozenbitarray('1'):
            self.round_trip(a)

    def test_subclass(self):
        class Foo(bitarray):
            pass

        a = Foo()
        code = self.get_code_string(a)
        self.assertEqual(code, "Foo()\n")
        b = eval(code)
        self.assertEqual(b, a)
        self.assertIs(type(b), type(a))

    def test_random(self):
        for n in range(150):
            self.round_trip(urandom(n))

    def test_file(self):
        with tempfile.TemporaryDirectory() as tmpdir:
            tmpfile = os.path.join(tmpdir, 'testfile')
            a = urandom_2(1000)
            with open(tmpfile, 'w') as fo:
                pprint(a, fo)
            with open(tmpfile, 'r') as fi:
                b = eval(fi.read())
            self.assertEqual(a, b)

    def test_errors(self):
        a = urandom(60)
        self.assertRaises(TypeError, pprint, a, group=6.0)
        self.assertRaises(TypeError, pprint, a, indent=0.0)
        self.assertRaises(TypeError, pprint, a, width=60.0)

# -----------------------------  strip()  -----------------------------------

class StripTests(unittest.TestCase, Util):

    def test_simple(self):
        self.assertRaises(TypeError, strip, '0110')
        self.assertRaises(TypeError, strip, bitarray(), 123)
        self.assertRaises(ValueError, strip, bitarray(), 'up')

        a = bitarray('00010110000')
        self.assertEQUAL(strip(a), bitarray('0001011'))
        self.assertEQUAL(strip(a, 'left'), bitarray('10110000'))
        self.assertEQUAL(strip(a, 'both'), bitarray('1011'))
        b = frozenbitarray('00010110000')
        c = strip(b, 'both')
        self.assertEqual(c, bitarray('1011'))
        self.assertIs(type(c), frozenbitarray)

    def test_zeros_ones(self):
        for _ in range(50):
            n = randrange(10)
            mode = choice(['left', 'right', 'both'])
            a = zeros(n)
            c = strip(a, mode)
            self.assertIs(type(c), bitarray)
            self.assertEqual(len(c), 0)
            self.assertEqual(a, zeros(n))

            b = frozenbitarray(a)
            c = strip(b, mode)
            self.assertIs(type(c), frozenbitarray)
            self.assertEqual(len(c), 0)

            a.setall(1)
            c = strip(a, mode)
            self.assertEqual(c, ones(n))

    def test_random(self):
        for a in self.randombitarrays():
            b = a.copy()
            f = frozenbitarray(a)
            s = a.to01()
            for mode, res in [
                    ('left',  bitarray(s.lstrip('0'), a.endian)),
                    ('right', bitarray(s.rstrip('0'), a.endian)),
                    ('both',  bitarray(s.strip('0'),  a.endian)),
            ]:
                c = strip(a, mode)
                self.assertEQUAL(c, res)
                self.assertIs(type(c), bitarray)
                self.assertEQUAL(a, b)

                c = strip(f, mode)
                self.assertEQUAL(c, res)
                self.assertIs(type(c), frozenbitarray)
                self.assertEQUAL(f, b)

    def test_one_set(self):
        for _ in range(10):
            n = randint(1, 10000)
            a = bitarray(n)
            a.setall(0)
            a[randrange(n)] = 1
            self.assertEqual(strip(a, 'both'), bitarray('1'))
            self.assertEqual(len(a), n)

# -----------------------------  count_n()  ---------------------------------

class CountN_Tests(unittest.TestCase, Util):

    @staticmethod
    def count_n(a, n):
        "return lowest index i for which a[:i].count() == n"
        i, j = n, a.count(1, 0, n)
        while j < n:
            j += a[i]
            i += 1
        return i

    def check_result(self, a, n, i, v=1):
        self.assertEqual(a.count(v, 0, i), n)
        if i == 0:
            self.assertEqual(n, 0)
        else:
            self.assertEqual(a[i - 1], v)

    def test_empty(self):
        a = bitarray()
        self.assertEqual(count_n(a, 0), 0)
        self.assertEqual(count_n(a, 0, 0), 0)
        self.assertEqual(count_n(a, 0, 1), 0)
        self.assertRaises(ValueError, count_n, a, 1)
        self.assertRaises(TypeError, count_n, '', 0)
        self.assertRaises(TypeError, count_n, a, 7.0)
        self.assertRaises(ValueError, count_n, a, 0, 2)
        self.assertRaisesMessage(ValueError, "n = 1 larger than bitarray "
                                 "length 0", count_n, a, 1)

    def test_simple(self):
        a = bitarray('111110111110111110111110011110111110111110111000')
        b = a.copy()
        self.assertEqual(len(a), 48)
        self.assertEqual(a.count(), 37)
        self.assertEqual(a.count(0), 11)

        self.assertEqual(count_n(a, 0), 0)
        self.assertEqual(count_n(a, 0, 0), 0)
        self.assertEqual(count_n(a, 2, 0), 12)
        self.assertEqual(count_n(a, 10, 0), 47)
        self.assertEqual(count_n(a, 20), 23)
        self.assertEqual(count_n(a, 20, 1), 23)
        self.assertEqual(count_n(a, 37), 45)
        # n < 0
        self.assertRaisesMessage(ValueError, "non-negative integer expected",
                                 count_n, a, -1)
        # n > len(a)
        self.assertRaisesMessage(ValueError, "n = 49 larger than bitarray "
                                 "length 48", count_n, a, 49)
        # n > a.count(0)
        self.assertRaisesMessage(ValueError, "n = 12 exceeds total count "
                                 "(a.count(0) = 11)", count_n, a, 12, 0)
        # n > a.count(1)
        self.assertRaisesMessage(ValueError, "n = 38 exceeds total count "
                                 "(a.count(1) = 37)", count_n, a, 38, 1)

        for v in 0, 1:
            for n in range(a.count(v) + 1):
                i = count_n(a, n, v)
                self.check_result(a, n, i, v)
                self.assertEqual(a[:i].count(v), n)
                self.assertEqual(i, self.count_n(a if v else ~a, n))
        self.assertEQUAL(a, b)

    def test_frozenbitarray(self):
        a = frozenbitarray('001111101111101111101111100111100')
        self.assertEqual(len(a), 33)
        self.assertEqual(a.count(), 24)
        self.assertEqual(count_n(a, 0), 0)
        self.assertEqual(count_n(a, 10), 13)
        self.assertEqual(count_n(a, 24), 31)
        self.assertRaises(ValueError, count_n, a, -1) # n < 0
        self.assertRaises(ValueError, count_n, a, 25) # n > a.count()
        self.assertRaises(ValueError, count_n, a, 34) # n > len(a)
        for n in range(25):
            self.check_result(a, n, count_n(a, n))

    def test_ones(self):
        n = randint(1, 100_000)
        a = ones(n)
        self.assertEqual(count_n(a, n), n)
        self.assertRaises(ValueError, count_n, a, 1, 0)
        self.assertRaises(ValueError, count_n, a, n + 1)
        for _ in range(20):
            i = randint(0, n)
            self.assertEqual(count_n(a, i), i)

    def test_one_set(self):
        n = randint(1, 100_000)
        a = zeros(n)
        self.assertEqual(count_n(a, 0), 0)
        self.assertRaises(ValueError, count_n, a, 1)
        for _ in range(20):
            a.setall(0)
            i = randrange(n)
            a[i] = 1
            self.assertEqual(count_n(a, 1), i + 1)
            self.assertRaises(ValueError, count_n, a, 2)

    def test_last(self):
        for N in range(1, 1000):
            a = zeros(N)
            a[-1] = 1
            self.assertEqual(count_n(a, 1), N)
            if N == 1:
                msg = "n = 2 larger than bitarray length 1"
            else:
                msg = "n = 2 exceeds total count (a.count(1) = 1)"
            self.assertRaisesMessage(ValueError, msg, count_n, a, 2)

    def test_primes(self):
        a = gen_primes(10_000)
        # there are 1229 primes below 10,000
        self.assertEqual(a.count(), 1229)
        for n, p in [(  10,   29),   # the 10th prime number is 29
                     ( 100,  541),   # the 100th prime number is 541
                     (1000, 7919)]:  # the 1000th prime number is 7919
            self.assertEqual(count_n(a, n) - 1, p)

    def test_large(self):
        for _ in range(100):
            N = randint(100_000, 250_000)
            a = bitarray(N)
            v = getrandbits(1)
            a.setall(not v)
            for _ in range(randrange(100)):
                a[randrange(N)] = v
            tc = a.count(v)      # total count
            i = count_n(a, tc, v)
            self.check_result(a, tc, i, v)
            n = tc + 1
            self.assertRaisesMessage(ValueError, "n = %d exceeds total count "
                                     "(a.count(%d) = %d)" % (n, v, tc),
                                     count_n, a, n, v)
            for _ in range(20):
                n = randint(0, tc)
                i = count_n(a, n, v)
                self.check_result(a, n, i, v)

# ---------------------------------------------------------------------------

class BitwiseCountTests(unittest.TestCase, Util):

    def test_count_byte(self):
        for i in range(256):
            a = bitarray(bytearray([i]))
            cnt = a.count()
            self.assertEqual(count_and(a, zeros(8)), 0)
            self.assertEqual(count_and(a, ones(8)), cnt)
            self.assertEqual(count_and(a, a), cnt)
            self.assertEqual(count_or(a, zeros(8)), cnt)
            self.assertEqual(count_or(a, ones(8)), 8)
            self.assertEqual(count_or(a, a), cnt)
            self.assertEqual(count_xor(a, zeros(8)), cnt)
            self.assertEqual(count_xor(a, ones(8)), 8 - cnt)
            self.assertEqual(count_xor(a, a), 0)

    def test_1(self):
        a = bitarray('001111')
        aa = a.copy()
        b = bitarray('010011')
        bb = b.copy()
        self.assertEqual(count_and(a, b), 2)
        self.assertEqual(count_or(a, b), 5)
        self.assertEqual(count_xor(a, b), 3)
        for f in count_and, count_or, count_xor:
            # not two arguments
            self.assertRaises(TypeError, f)
            self.assertRaises(TypeError, f, a)
            self.assertRaises(TypeError, f, a, b, 3)
            # wrong argument types
            self.assertRaises(TypeError, f, a, '')
            self.assertRaises(TypeError, f, '1', b)
            self.assertRaises(TypeError, f, a, 4)
        self.assertEQUAL(a, aa)
        self.assertEQUAL(b, bb)

        b.append(1)
        for f in count_and, count_or, count_xor:
            self.assertRaises(ValueError, f, a, b)
            self.assertRaises(ValueError, f,
                              bitarray('110', 'big'),
                              bitarray('101', 'little'))

    def test_frozen(self):
        a = frozenbitarray('001111')
        b = frozenbitarray('010011')
        self.assertEqual(count_and(a, b), 2)
        self.assertEqual(count_or(a, b), 5)
        self.assertEqual(count_xor(a, b), 3)

    def test_random(self):
        for _ in range(100):
            n = randrange(1000)
            a = urandom_2(n)
            b = urandom(n, a.endian)
            self.assertEqual(count_and(a, b), (a & b).count())
            self.assertEqual(count_or(a, b),  (a | b).count())
            self.assertEqual(count_xor(a, b), (a ^ b).count())

    def test_misc(self):
        for a in self.randombitarrays():
            n = len(a)
            b = urandom(n, a.endian)
            # any and
            self.assertEqual(any(a & b), count_and(a, b) > 0)
            self.assertEqual(any_and(a, b), any(a & b))
            # any or
            self.assertEqual(any(a | b), count_or(a, b) > 0)
            self.assertEqual(any(a | b), any(a) or any(b))
            # any xor
            self.assertEqual(any(a ^ b), count_xor(a, b) > 0)
            self.assertEqual(any(a ^ b), a != b)

            # all and
            self.assertEqual(all(a & b), count_and(a, b) == n)
            self.assertEqual(all(a & b), all(a) and all(b))
            # all or
            self.assertEqual(all(a | b), count_or(a, b) == n)
            # all xor
            self.assertEqual(all(a ^ b), count_xor(a, b) == n)
            self.assertEqual(all(a ^ b), a == ~b)

# ---------------------------  any_and()  -----------------------------------

class BitwiseAnyTests(unittest.TestCase, Util):

    def test_basic(self):
        a = frozenbitarray('0101')
        b = bitarray('0111')
        self.assertTrue(any_and(a, b))
        self.assertRaises(TypeError, any_and)
        self.assertRaises(TypeError, any_and, a, 4)
        b.append(1)
        self.assertRaises(ValueError, any_and, a, b)
        self.assertRaises(ValueError, any_and,
                          bitarray('01', 'little'),
                          bitarray('11', 'big'))

    def test_overlap(self):
        n = 100
        for _ in range(500):
            i1 = randint(0, n)
            j1 = randint(i1, n)
            r1 = range(i1, j1)

            i2 = randint(0, n)
            j2 = randint(i2, n)
            r2 = range(i2, j2)

            # test if ranges r1 and r2 overlap
            res1 = bool(r1) and bool(r2) and (i2 in r1 or i1 in r2)
            res2 = bool(set(r1) & set(r2))
            self.assertEqual(res1, res2)

            a1, a2 = bitarray(n), bitarray(n)
            a1[i1:j1] = a2[i2:j2] = 1
            self.assertEqual(any_and(a1, a2), res1)

    def test_common(self):
        n = 100
        for _ in range(500):
            s1 = self.random_slice(n)
            s2 = self.random_slice(n)
            r1 = range(n)[s1]
            r2 = range(n)[s2]
            # test if ranges r1 and r2 have common items
            a1, a2 = bitarray(n), bitarray(n)
            a1[s1] = a2[s2] = 1
            self.assertEqual(any_and(a1, a2), bool(set(r1) & set(r2)))

    def check(self, a, b):
        r = any_and(a, b)
        self.assertIs(type(r), bool)
        self.assertEqual(r, any_and(b, a))  # symmetry
        self.assertEqual(r, any(a & b))
        self.assertEqual(r, (a & b).any())
        self.assertEqual(r, count_and(a, b) > 0)

    def test_explicit(self):
        for a, b , res in [
                ('', '', False),
                ('0', '1', False),
                ('0', '0', False),
                ('1', '1', True),
                ('00011', '11100', False),
                ('00001011 1', '01000100 1', True)]:
            a = bitarray(a)
            b = bitarray(b)
            self.assertIs(any_and(a, b), res)
            self.check(a, b)

    def test_random(self):
        for a in self.randombitarrays():
            n = len(a)
            b = urandom(n, a.endian)
            self.check(a, b)

    def test_one(self):
        for n in range(1, 300):
            a = zeros(n)
            b = urandom(n)
            i = randrange(n)
            a[i] = 1
            self.assertEqual(b[i], any_and(a, b))

# ----------------------------  subset()  -----------------------------------

class SubsetTests(unittest.TestCase, Util):

    def test_basic(self):
        a = frozenbitarray('0101')
        b = bitarray('0111')
        self.assertTrue(subset(a, b))
        self.assertFalse(subset(b, a))
        self.assertRaises(TypeError, subset)
        self.assertRaises(TypeError, subset, a, '')
        self.assertRaises(TypeError, subset, '1', b)
        self.assertRaises(TypeError, subset, a, 4)
        b.append(1)
        self.assertRaises(ValueError, subset, a, b)
        self.assertRaises(ValueError, subset,
                          bitarray('01', 'little'),
                          bitarray('11', 'big'))

    def check(self, a, b, res):
        r = subset(a, b)
        self.assertIs(type(r), bool)
        self.assertEqual(r, res)
        self.assertEqual(a | b == b, res)
        self.assertEqual(a & b == a, res)

    def test_true(self):
        for a, b in [('', ''), ('0', '1'), ('0', '0'), ('1', '1'),
                     ('000', '111'), ('0101', '0111'),
                     ('000010111', '010011111')]:
            self.check(bitarray(a), bitarray(b), True)

    def test_false(self):
        for a, b in [('1', '0'), ('1101', '0111'),
                     ('0000101111', '0100111011')]:
            self.check(bitarray(a), bitarray(b), False)

    def test_random(self):
        for a in self.randombitarrays(start=1):
            b = a.copy()
            # we set one random bit in b to 1, so a is always a subset of b
            b[randrange(len(a))] = 1
            self.check(a, b, True)
            # but b is only a subset when they are equal
            self.check(b, a, a == b)
            # we set all bits in a, which ensures that b is a subset of a
            a.setall(1)
            self.check(b, a, True)

# -------------------------  correspond_all()  ------------------------------

class CorrespondAllTests(unittest.TestCase):

    def test_basic(self):
        a = frozenbitarray('0101')
        b = bitarray('0110')
        self.assertEqual(correspond_all(a, b), (1, 1, 1, 1))
        self.assertRaises(TypeError, correspond_all)
        b.append(1)
        self.assertRaises(ValueError, correspond_all, a, b)
        self.assertRaises(ValueError, correspond_all,
                          bitarray('01', 'little'),
                          bitarray('11', 'big'))

    def test_explicit(self):
        for a, b, res in [
                ('', '', (0, 0, 0, 0)),
                ('0000011111',
                 '0000100111', (4, 1, 2, 3)),
            ]:
            self.assertEqual(correspond_all(bitarray(a), bitarray(b)), res)

    def test_random(self):
        for _ in range(100):
            n = randrange(3000)
            a = urandom_2(n)
            b = urandom(n, a.endian)
            res = correspond_all(a, b)
            self.assertEqual(res[0], count_and(~a, ~b))
            self.assertEqual(res[1], count_and(~a, b))
            self.assertEqual(res[2], count_and(a, ~b))
            self.assertEqual(res[3], count_and(a, b))

            self.assertEqual(res[0], n - count_or(a, b))
            self.assertEqual(res[1] + res[2], count_xor(a, b))
            self.assertEqual(sum(res), n)

# -----------------------------  byteswap()  --------------------------------

@unittest.skipIf(is_pypy, "skip test on PyPy")
class ByteSwapTests(unittest.TestCase):

    def test_basic_bytearray(self):
        a = bytearray(b"ABCD")
        byteswap(a, 2)
        self.assertEqual(a, bytearray(b"BADC"))
        byteswap(a)
        self.assertEqual(a, bytearray(b"CDAB"))

        a = bytearray(b"ABCDEF")
        byteswap(a, 3)
        self.assertEqual(a, bytearray(b"CBAFED"))
        byteswap(a, 1)
        self.assertEqual(a, bytearray(b"CBAFED"))

    def test_basic_bitarray(self):
        a = bitarray("11110000 01010101")
        byteswap(a)
        self.assertEqual(a, bitarray("01010101 11110000"))

        a = bitarray("01111000 1001")
        b = a.copy()
        a.tobytes()  # clear padbits
        byteswap(a)
        self.assertEqual(a, bitarray("10010000 0111"))
        byteswap(a)
        self.assertEqual(a, b)

    def test_basic_array(self):
        r = os.urandom(64)
        for typecode in array.typecodes:
            # type code 'u' is deprecated and will be removed in Python 3.16
            if typecode in ('u', 'Zf', 'Zd'):
                continue
            a = array.array(typecode, r)
            self.assertEqual(len(a) * a.itemsize, 64)
            a.byteswap()
            byteswap(a, a.itemsize)
            self.assertEqual(a.tobytes(), r)

    def test_empty(self):
        a = bytearray()
        byteswap(a)
        self.assertEqual(a, bytearray())
        for n in range(10):
            byteswap(a, n)
            self.assertEqual(a, bytearray())

    def test_one_byte(self):
        a = bytearray(b'\xab')
        byteswap(a)
        self.assertEqual(a, bytearray(b'\xab'))
        for n in range(2):
            byteswap(a, n)
            self.assertEqual(a, bytearray(b'\xab'))

    def test_errors(self):
        # buffer not writable
        for a in b"AB", frozenbitarray(16):
            self.assertRaises(BufferError, byteswap, a)

        a = bytearray(b"ABCD")
        b = bitarray(32)
        for n in -1, 3, 5, 6:
            # byte size not multiple of n
            self.assertRaises(ValueError, byteswap, a, n)
            self.assertRaises(ValueError, byteswap, b, n)

    def test_range(self):
        for n in range(20):
            for m in range(20):
                r = os.urandom(m * n)
                a = bytearray(r)
                byteswap(a, n)
                lst = []
                for i in range(m):
                    x = r[i * n:i * n + n]
                    lst.extend(x[::-1])
                self.assertEqual(a, bytearray(lst))

    def test_reverse_bytearray(self):
        for n in range(100):
            r = os.urandom(n)
            a = bytearray(r)
            byteswap(a)
            self.assertEqual(a, bytearray(r[::-1]))

    def test_reverse_bitarray(self):
        for n in range(100):
            a = urandom(8 * n)
            b = a.copy()
            byteswap(a)
            a.bytereverse()
            self.assertEqual(a, b[::-1])

# ------------------------------  parity()  ---------------------------------

class ParityTests(unittest.TestCase):

    def test_explicit(self):
        for s, res in [('', 0), ('1', 1), ('0010011', 1), ('10100110', 0)]:
            self.assertIs(parity(bitarray(s)), res)
            self.assertIs(parity(frozenbitarray(s)), res)

    def test_zeros_ones(self):
        for n in range(2000):
            self.assertEqual(parity(zeros(n)), 0)
            self.assertEqual(parity(ones(n)), n % 2)

    def test_random(self):
        endian = choice(ENDIANS)
        a = bitarray(endian=endian)
        par = 0
        for i in range(2000):
            self.assertEqual(parity(a), par)
            self.assertEqual(par, a.count() % 2)
            self.assertEqual(a.endian, endian)
            self.assertEqual(len(a), i)
            v = getrandbits(1)
            a.append(v)
            par ^= v

    def test_wrong_args(self):
        self.assertRaises(TypeError, parity, '')
        self.assertRaises(TypeError, parity, 1)
        self.assertRaises(TypeError, parity)
        self.assertRaises(TypeError, parity, bitarray("110"), 1)

# ----------------------------  sum_indices()  ------------------------------

class SumIndicesTests(unittest.TestCase):

    # Additional tests in: devel/test_sum_indices.py

    def test_explicit(self):
        endian = choice(ENDIANS)
        a = bitarray("01100111 1101", endian)
        self.assertEqual(sum_indices(a, mode=1),  49)
        self.assertEqual(sum_indices(a, mode=2), 381)
        a = frozenbitarray("00001111", endian)
        self.assertEqual(sum_indices(a, mode=1),  22)
        self.assertEqual(sum_indices(a, mode=2), 126)

    def test_wrong_args(self):
        S = sum_indices
        self.assertRaises(TypeError, S, '')
        self.assertRaises(TypeError, S, 1.0)
        self.assertRaises(TypeError, S)
        for mode in -1, 0, 3, 4:
            self.assertRaises(ValueError, S, bitarray("110"), mode)

    def test_ones(self):
        a = bitarray(endian=choice(ENDIANS))
        sm1 = sm2 = 0
        for i in range(100):
            a.append(1)
            sm1 += i
            sm2 += i * i
            self.assertEqual(sum_indices(a, 1), sm1)
            self.assertEqual(sum_indices(a, 2), sm2)

    def check_sparse(self, n, k, mode):
        a = zeros(n, choice(ENDIANS))
        indices = sample(range(n), k)
        a[indices] = 1
        res = sum(indices) if mode == 1 else sum(i * i for i in indices)
        self.assertEqual(a.count(), k)
        self.assertEqual(sum_indices(a, mode), res)

    def test_small(self):
        for mode in 1, 2:
            for a in bitarray(), bitarray("0"), bitarray("1"):
                self.assertEqual(sum_indices(a, mode), 0)

            for n in range(2, 50):
                k = randrange(n // 2)
                self.check_sparse(n, k, mode)

    def test_sparse(self):
        for _ in range(10):
            n = randrange(2, 1_500_000)  # below and above block size
            k = randrange(min(100, n // 2))
            mode = randint(1, 2)
            self.check_sparse(n, k, mode)

# ---------------------------------------------------------------------------

class XoredIndicesTests(unittest.TestCase, Util):

    def test_explicit(self):
        for s, r in [("", 0), ("0", 0), ("1", 0), ("11", 1),
                     ("011", 3), ("001", 2), ("0001100", 7),
                     ("01100111 1101", 13)]:
            for a in [bitarray(s, choice(ENDIANS)),
                      frozenbitarray(s, choice(ENDIANS))]:
                self.assertEqual(xor_indices(a), r)

    def test_wrong_args(self):
        X = xor_indices
        self.assertRaises(TypeError, X, '')
        self.assertRaises(TypeError, X, 1)
        self.assertRaises(TypeError, X)
        self.assertRaises(TypeError, X, bitarray("110"), 1)

    def test_ones(self):
        # OEIS A003815
        lst = [0, 1, 3, 0, 4, 1, 7, 0, 8, 1, 11, 0, 12, 1, 15, 0, 16, 1, 19]
        self.assertEqual([xor_indices(ones(i)) for i in range(1, 20)], lst)
        a = bitarray()
        x = 0
        for i in range(1000):
            a.append(1)
            x ^= i
            self.assertEqual(xor_indices(a), x)
            if i < 19:
                self.assertEqual(lst[i], x)

    def test_primes(self):
        # OEIS A126084
        lst = [0, 2, 1, 4, 3, 8, 5, 20, 7, 16, 13, 18, 55, 30, 53, 26, 47]
        primes = gen_primes(1000)
        x = 0
        for i, p in enumerate(primes.search(1)):
            self.assertEqual(xor_indices(primes[:p]), x)
            if i < 17:
                self.assertEqual(lst[i], x)
            x ^= p

    def test_large_random(self):
        n = 10_037
        for a in [urandom_2(n), frozenbitarray(urandom_2(n))]:
            res = reduce(operator.xor, (i for i, v in enumerate(a) if v))
            b = a.copy()
            self.assertEqual(xor_indices(a), res)
            self.assertEqual(a, b)

    def test_random(self):
        for a in self.randombitarrays():
            c = 0
            for i, v in enumerate(a):
                c ^= i * v
            self.assertEqual(xor_indices(a), c)

    def test_flips(self):
        a = bitarray(128)
        c = 0
        for _ in range(1000):
            self.assertEqual(xor_indices(a), c)
            i = randrange(len(a))
            a.invert(i)
            c ^= i

    def test_error_correct(self):
        parity_bits = [1, 2, 4, 8, 16, 32, 64, 128]  # parity bit positions
        a = urandom(256)
        a[parity_bits] = 0
        c = xor_indices(a)
        # set parity bits such that block is well prepared
        a[parity_bits] = int2ba(c, length=8, endian="little")
        for i in range(0, 256):
            self.assertEqual(xor_indices(a), 0)  # ensure well prepared
            a.invert(i)
            self.assertEqual(xor_indices(a), i)  # index of the flipped bit!
            a.invert(i)


# -------------------------   run-length codec   ----------------------------

class RL_Util:

    @staticmethod
    def runs(a):
        "return number of uninterrupted intervals of 1s and 0s"
        n = len(a)
        if n < 2:
            return n
        return count_xor(a[:-1], a[1:]) + 1

    def random_runs(self, n, k):
        "generate random bitarray of length `n` with `k` runs."
        self.assertTrue(n == k == 0 or 1 <= k <= n)
        a = bitarray(n)
        if n == 0:
            return a
        vi = getrandbits(1)

        # Create sorted sample from `range(1, n)`, with `n` last and
        # with a total of `k` items.
        lst = sample(range(1, n), k - 1)
        lst.append(n)
        lst.sort()

        start = 0
        for stop in lst:
            a[start:stop] = vi
            vi = not vi
            start = stop
        return a


class RL_Tests(unittest.TestCase, Util, RL_Util):

    def test_explicit(self):
        for s, b in [
                ("",    b'0\x00'),
                ("0",   b'0\x01\x01'),
                ("1",   b'1\x01\x01'),
                ("10",  b'1\x02\x01\x01'),
                ("01",  b'0\x02\x01\x01'),
                ("1" + 63 * "0", b'1\x40\x01\x3f'),
                ("0" + 63 * "1", b'0\x40\x01\x3f'),
        ]:
            a = bitarray(s)
            self.assertEqual(rl_encode(a), b)
            self.assertEqual(rl_decode(b), a)
            it = iter(b)
            self.assertEqual(rl_decode(it), a)
            self.assertRaises(StopIteration, next, it)
            it = iter(b + b'XYZ')
            self.assertEqual(rl_decode(it), a)
            self.assertEqual(next(it), ord(b'X'))

    def test_decode_errors(self):
        # incomplete stream
        for b in [
                b'',           # no first bit
                b'0',          # no uleb128 encoded nbits
                b'1\x80',      # nbits uleb128 encoding incomplete
                b'0\x01',      # run length incomplete
                b'1\x02\x01',  # run length incomplete
        ]:
            self.assertRaises(StopIteration, rl_decode, b)

        # invalid first bit
        for b in b'\x00', b'\x01', b'2':
            msg = "invalid first bit: 0x%02x" % ord(b)
            self.assertRaisesMessage(ValueError, msg, rl_decode, b)

        # run length cannot be zero
        for b in b'0\x01\x00\x01', b'1\xff\x0a\x01\x00':
            msg = "run length cannot be zero"
            self.assertRaisesMessage(ValueError, msg, rl_decode, b)

        # sequence exceeds nbits
        for b, nrun, nrem in [
                (b'1\x01\x02',      2,  1),
                (b'1\x30\x2f\x02',  2,  1),
                (b'0\x40\x01\x40', 64, 63),
        ]:
            msg = "run length %d exceeds remaining length %d" % (nrun, nrem)
            self.assertRaisesMessage(ValueError, msg, rl_decode, b)

    def test_decode_ambiguity(self):
        # For an empty bitarray either first-bit byte works.
        for b in b'0\x00', b'1\x00':
            self.assertEqual(rl_decode(b), bitarray())

        # ULEB128 permits overlong representations.
        for b in b'0\x00', b'0\x80\x00':
            self.assertEqual(rl_decode(b), bitarray())

    def test_decode_endian(self):
        b = b'0\x02\x01\x01'
        for endian in OPT_ENDIANS:
            a = rl_decode(b, endian=endian)
            self.assertEqual(a.endian, endian or get_default_endian())
            self.assertEqual(a, bitarray("01"))

    def test_decode_types(self):
        b = b'0\x02\x01\x01'
        for c in b, bytearray(b), list(b), iter(b):
            a = rl_decode(c)
            self.assertIs(type(a), bitarray)
            self.assertEqual(a.to01(), '01')

    def test_encode_types(self):
        a = frozenbitarray("00001111 11111111 11110000 000")
        self.assertEqual(len(a), 0x1b)
        b = rl_encode(a)
        self.assertIs(type(b), bytes)
        self.assertEqual(b, b'0\x1b\x04\x10\x07')

    def test_intervals(self):
        a = bitarray("00000011111111111111110000000111000000")
        b = bytearray([ord("0")])
        b.extend(uleb128_encode(len(a)))
        for unused_value, start, stop in intervals(a):
            b.extend(uleb128_encode(stop - start))
        self.assertEqual(rl_decode(b), a)
        self.assertEqual(b, rl_encode(a))

    def test_concatenate(self):
        N = 100
        arrays = [urandom(randrange(5)) for _ in range(N)]
        it = iter(b''.join(rl_encode(a) for a in arrays))
        self.assertEqual([rl_decode(it) for _ in range(N)], arrays)
        self.assertRaises(StopIteration, next, it)

    def test_random(self):
        for _ in range(10):
            n = randrange(1, 100)
            k = randint(1, max(1, n // 2))
            a = self.random_runs(n, k)

            b = bytearray([ord("0") + a[0]])
            b.extend(uleb128_encode(len(a)))
            for _, start, stop in intervals(a):
                b.extend(uleb128_encode(stop - start))

            self.assertEqual(rl_decode(b), a)
            self.assertEqual(b, rl_encode(a))


# ------------------   intervals of uninterrupted runs   --------------------

class IntervalsTests(unittest.TestCase, Util, RL_Util):

    def test_explicit(self):
        for s, lst in [
                ('', []),
                ('0', [(0, 0, 1)]),
                ('1', [(1, 0, 1)]),
                ('00111100 0000011',
                 [(0, 0, 2), (1, 2, 6), (0, 6, 13), (1, 13, 15)]),
            ]:
            a = bitarray(s)
            self.assertEqual(list(intervals(a)), lst)
            self.assertEqual(self.runs(a), len(lst))

    def test_uniform(self):
        for n in range(1, 100):
            a = bitarray(n, choice(ENDIANS))
            for v in 0, 1:
                a.setall(v)
                self.assertEqual(list(intervals(a)), [(v, 0, n)])
                self.assertEqual(self.runs(a), 1)

    def test_random(self):
        for _ in range(10):
            n = randrange(2, 500)
            k = randint(1, n // 2)
            a = self.random_runs(n, k)

            # Reconstruct `a` from its intervals, starting with random data.
            b = urandom(n)
            for value, start, stop in intervals(a):
                self.assertIs(type(value), int)
                self.assertTrue(0 <= start < stop <= n)
                b[start:stop] = value
            self.assertEqual(a, b)

    def test_list_runs(self):
        for a in self.randombitarrays():
            # list of length of runs of alternating bits
            alt_runs = [stop - start for _, start, stop in intervals(a)]
            self.assertEqual(len(alt_runs), self.runs(a))

            b = bitarray()
            v = a[0] if a else None  # value of first run
            for length in alt_runs:
                self.assertTrue(length > 0)
                b.extend(length * bitarray([v]))
                v = not v
            self.assertEqual(a, b)


# --------------------------  ba2hex()  hex2ba()  ---------------------------

class HexlifyTests(unittest.TestCase, Util):

    def test_explicit(self):
        data = [ #                  little   big
            ('',                    '',      ''),
            ('1000',                '1',     '8'),
            ('0101 0110',           'a6',    '56'),
            ('0100 1001 1101',      '29b',   '49d'),
            ('0000 1100 1110 1111', '037f',  '0cef'),
        ]
        for bs, hex_le, hex_be in data:
            a_be = bitarray(bs, 'big')
            a_le = bitarray(bs, 'little')
            self.assertEQUAL(hex2ba(hex_be, 'big'), a_be)
            self.assertEQUAL(hex2ba(hex_le, 'little'), a_le)
            self.assertEqual(ba2hex(a_be), hex_be)
            self.assertEqual(ba2hex(a_le), hex_le)

    def test_ba2hex_group(self):
        a = bitarray('1000 0000 0101 1111', 'little')
        self.assertEqual(ba2hex(a), "10af")
        self.assertEqual(ba2hex(a, 0), "10af")
        self.assertEqual(ba2hex(a, 1, ""), "10af")
        self.assertEqual(ba2hex(a, 1), "1 0 a f")
        self.assertEqual(ba2hex(a, group=2), "10 af")
        self.assertEqual(ba2hex(a, 2, "-"), "10-af")
        self.assertEqual(ba2hex(a, group=3, sep="_"), "10a_f")
        self.assertEqual(ba2hex(a, 3, sep=", "), "10a, f")

    def test_ba2hex_errors(self):
        self.assertRaises(TypeError, ba2hex)
        self.assertRaises(TypeError, ba2hex, None)
        self.assertRaises(TypeError, ba2hex, '101')

        # length not multiple of 4
        self.assertRaises(ValueError, ba2hex, bitarray('10'))

        a = bitarray('1000 0000 0101 1111', 'little')
        self.assertRaises(ValueError, ba2hex, a, -1)
        self.assertRaises(ValueError, ba2hex, a, group=-1)
        # sep not str
        self.assertRaises(TypeError, ba2hex, a, 1, b" ")
        # embedded null character in sep
        self.assertRaises(ValueError, ba2hex, a, 2, " \0")

    def test_hex2ba_whitespace(self):
        self.assertEqual(hex2ba("F1 FA %s f3 c0" % whitespace),
                         bitarray("11110001 11111010 11110011 11000000"
                                  if get_default_endian() == "big" else
                                  "11111000 11110101 11111100 00110000"))
        self.assertEQUAL(hex2ba(b' a F ', 'big'),
                         bitarray('1010 1111', 'big'))
        self.assertEQUAL(hex2ba(860 * " " + '0  1D' + 590 * " ", 'little'),
                         bitarray('0000 1000 1011', 'little'))

    def test_hex2ba_errors(self):
        self.assertRaises(TypeError, hex2ba, 0)
        self.assertRaises(TypeError, hex2ba, "F", 1)
        self.assertRaises(ValueError, hex2ba, "F", "foo")

        for s in '01a7g89', '0\u20ac', '0 \0', b'\x00':
            self.assertRaises(ValueError, hex2ba, s)

        for s in 'g', 'ag', 'aag', 'aaaga', 'ag':
            msg = "invalid digit found for base16, got 'g' (0x67)"
            self.assertRaisesMessage(ValueError, msg, hex2ba, s, 'big')

    def test_hex2ba_types(self):
        for c in 'e', 'E', b'e', b'E', bytearray(b'e'), bytearray(b'E'):
            a = hex2ba(c, "big")
            self.assertEqual(a.to01(), '1110')
            self.assertEqual(a.endian, 'big')
            self.assertIs(type(a), bitarray)

    def test_random(self):
        for _ in range(100):
            endian = choice(OPT_ENDIANS)
            a = urandom(4 * randrange(100), endian)
            s = ba2hex(a, group=randrange(10), sep=choice(whitespace))
            b = hex2ba(s, endian)
            self.assertEqual(b.endian, endian or get_default_endian())
            self.assertEqual(a, b)
            self.check_obj(b)

    def test_hexdigits(self):
        a = hex2ba(hexdigits)
        self.assertEqual(len(a), 4 * len(hexdigits))
        self.assertIs(type(a), bitarray)
        self.check_obj(a)

        t = ba2hex(a)
        self.assertEqual(t, hexdigits.lower())
        self.assertIs(type(t), str)
        self.assertEQUAL(a, hex2ba(t))

    def test_binascii(self):
        a = urandom(80, 'big')
        s = binascii.hexlify(a.tobytes()).decode()
        self.assertEqual(ba2hex(a), s)
        b = bitarray(binascii.unhexlify(s), endian='big')
        self.assertEQUAL(hex2ba(s, 'big'), b)

# --------------------------  ba2base()  base2ba()  -------------------------

class BaseTests(unittest.TestCase, Util):

    def test_explicit(self):
        data = [ #              n  little   big
            ('',                2, '',      ''),
            ('1 0 1',           2, '101',   '101'),
            ('11 01 00',        4, '320',   '310'),
            ('111 001',         8, '74',    '71'),
            ('1111 0001',      16, 'f8',    'f1'),
            ('11111 00001',    32, '7Q',    '7B'),
            ('111111 000001',  64, '/g',    '/B'),
        ]
        for bs, n, s_le, s_be in data:
            a_le = bitarray(bs, 'little')
            a_be = bitarray(bs, 'big')
            self.assertEQUAL(base2ba(n, s_le, 'little'), a_le)
            self.assertEQUAL(base2ba(n, s_be, 'big'),    a_be)
            self.assertEqual(ba2base(n, a_le), s_le)
            self.assertEqual(ba2base(n, a_be), s_be)

    def test_base2ba_types(self):
        for c in '7', b'7', bytearray(b'7'):
            a = base2ba(32, c)
            self.assertEqual(a.to01(), '11111')
            self.assertIs(type(a), bitarray)

    def test_base2ba_whitespace(self):
        self.assertEqual(base2ba(8, bytearray(b"17 0"), "little"),
                         bitarray("100 111 000"))
        self.assertEqual(base2ba(32, "7 A"), bitarray("11111 00000"))
        self.assertEqual(base2ba(64, b"A /"), bitarray("000000 111111"))
        for n in 2, 4, 8, 16, 32, 64:
            a = base2ba(n, whitespace)
            self.assertEqual(a, bitarray())
            a = urandom(60)
            c = list(ba2base(n, a))
            for _ in range(randrange(80)):
                c.insert(randint(0, len(c)), choice(whitespace))
            s = ''.join(c)
            self.assertEqual(base2ba(n, s), a)

    def test_ba2base_group(self):
        a = bitarray("001 011 100 111", "little")
        self.assertEqual(ba2base(8, a, 3), "461 7")
        self.assertEqual(ba2base(8, a, group=2), "46 17")
        self.assertEqual(ba2base(8, a, sep="_", group=2), "46_17")
        self.assertEqual(ba2base(8, a, 2, sep="."), "46.17")
        for n, s, group, sep, res in [
                (2, '10100', 2, '-', '10-10-0'),
                (4, '10 11 00 01', 1, "_", "2_3_0_1"),
                (8, "101 100 011 101 001 010", 3, "  ", "543  512"),
                (8, "101 100 011 101 001 010", 3, "", "543512"),
                (16, '1011 0001 1101 1010 1111', 4, "+", "b1da+f"),
                (32, "10110 00111 01101 01111", 2, ", ", "WH, NP"),
                (64, "101100 011101 101011 111110 101110", 2, ".", "sd.r+.u"),
                ]:
            a = bitarray(s, "big")
            s = ba2base(n, a, group, sep)
            self.assertIs(type(s), str)
            self.assertEqual(s, res)

    def test_empty(self):
        for n in 2, 4, 8, 16, 32, 64:
            a = base2ba(n, '')
            self.assertEqual(a, bitarray())
            self.assertEqual(ba2base(n, a), '')

    def test_invalid_characters(self):
        for n, s in ((2, '2'), (4, '4'), (8, '8'), (16, 'g'), (32, '8'),
                     (32, '1'), (32, 'a'), (64, '-'), (64, '_')):
            msg = ("invalid digit found for base%d, "
                   "got '%s' (0x%02x)" % (n, s, ord(s)))
            self.assertRaisesMessage(ValueError, msg, base2ba, n, s)

        for n in 2, 4, 8, 16, 32, 64:
            for s in '_', '@', '[', '\u20ac', '\0',  b'\0', b'\x80', b'\xff':
                self.assertRaises(ValueError, base2ba, n, s)
            msg = "invalid digit found for base%d, got '{' (0x7b)" % n
            self.assertRaisesMessage(ValueError, msg, base2ba, n, '{')

    def test_invalid_args(self):
        a = bitarray()
        self.assertRaises(TypeError, ba2base, None, a)
        self.assertRaises(TypeError, base2ba, None, '')
        self.assertRaises(TypeError, ba2base, 16.0, a)
        self.assertRaises(TypeError, base2ba, 16.0, '')
        self.assertRaises(TypeError, ba2base, 32, None)
        self.assertRaises(TypeError, base2ba, 32, None)

        for values, msg in [
                ([-1023, -16, -1, 0, 3, 5, 31, 48, 63, 129, 511, 4123,
                  1, 128, 256, 512, 1024, 2048, 4096, 8192],
                 "base must be 2, 4, 8, 16, 32 or 64")]:
            for i in values:
                self.assertRaisesMessage(ValueError, msg, ba2base, i, a)
                self.assertRaisesMessage(ValueError, msg, base2ba, i, '')

        a = bitarray(29)
        for m in range(2, 7):
            msg = "bitarray length 29 not multiple of %d" % m
            self.assertRaisesMessage(ValueError, msg, ba2base, 1 << m, a)

    def test_hexadecimal(self):
        a = base2ba(16, 'F61', 'big')
        self.assertEqual(a, bitarray('1111 0110 0001'))
        self.assertEqual(ba2base(16, a), 'f61')

        for n in range(50):
            s = ''.join(choices(hexdigits, k=n))
            endian = choice(ENDIANS)
            a = base2ba(16, s, endian)
            self.assertEQUAL(a, hex2ba(s, endian))
            self.assertEqual(ba2base(16, a), ba2hex(a))

    def test_base32(self):
        msg = os.urandom(randint(10, 100) * 5)
        s = base64.b32encode(msg).decode()
        a = base2ba(32, s, 'big')
        self.assertEqual(a.tobytes(), msg)
        self.assertEqual(ba2base(32, a), s)
        self.assertEqual(base64.b32decode(s), msg)

    def test_base64(self):
        msg = os.urandom(randint(10, 100) * 3)
        s = base64.standard_b64encode(msg).decode()
        a = base2ba(64, s, 'big')
        self.assertEqual(a.tobytes(), msg)
        self.assertEqual(ba2base(64, a), s)
        self.assertEqual(base64.standard_b64decode(s), msg)

    def test_primes(self):
        primes = gen_primes(60, odd=True)
        base_2 = primes.to01()
        for n, endian, rep in [
                ( 2, "little", base_2),
                ( 2, "big",    base_2),
                ( 4, "little", "232132030132012122122010132110"),
                ( 4, "big",    "131231030231021211211020231220"),
                ( 8, "little", "65554155441515405550"),
                ( 8, "big",    "35551455114545105550"),
                (16, "little", "e6bc4b46a921d61"),
                (16, "big",    "76d32d265948b68"),
                (32, "little", "O3SJLSJTSI3C"),
                (32, "big",    "O3JS2JSZJC3I"),
                (64, "little", "utMtkppEtF"),
                (64, "big",    "dtMtJllIto"),
        ]:
            a = bitarray(primes, endian)
            s = ba2base(n, a)
            self.assertIs(type(s), str)
            self.assertEqual(s, rep)
            b = base2ba(n, rep, endian)
            self.assertEqual(b, a)
            self.assertIs(type(b), bitarray)
            self.assertEqual(b.endian, endian)

    alphabets = [
    #    m   n  alphabet
        (1,  2, '01'),
        (2,  4, '0123'),
        (3,  8, '01234567'),
        (4, 16, '0123456789abcdef'),
        (4, 16, '0123456789ABCDEF'),
        (5, 32, 'ABCDEFGHIJKLMNOPQRSTUVWXYZ234567'),
        (6, 64, 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdef'
                'ghijklmnopqrstuvwxyz0123456789+/'),
    ]

    def test_alphabets(self):
        for m, n, alphabet in self.alphabets:
            self.assertEqual(1 << m, n)
            self.assertEqual(len(alphabet), n)
            for i, c in enumerate(alphabet):
                endian = choice(ENDIANS)
                self.assertEqual(ba2int(base2ba(n, c, endian)), i)
                if m == 4 and c in "ABCDEF":
                    c = chr(ord(c) + 32)
                self.assertEqual(ba2base(n, int2ba(i, m, endian)), c)

    def test_not_alphabets(self):
        for m, n, alphabet in self.alphabets:
            for i in range(256):
                c = chr(i)
                if c in alphabet or c.isspace():
                    continue
                if n == 16 and c in hexdigits:
                    continue
                self.assertRaises(ValueError, base2ba, n, c)

    def test_random(self):
        for _ in range(100):
            m = randint(1, 6)
            a = urandom_2(m * randrange(100))
            n = 1 << m
            s = ba2base(n, a, group=randrange(10), sep=randrange(5) * " ")
            if m == 4 and getrandbits(1):
                s = s.upper()
            if getrandbits(1):
                s = s.encode()
            b = base2ba(n, s, a.endian)
            self.assertEQUAL(a, b)
            self.check_obj(b)

# --------------------------- sparse compression ----------------------------

class SC_Util:

    def get_max_pop(self, n, m=256):
        """
        Return the maximal population for which one type `n` block is
        preferred over `m` blocks of type `n-1`, or 0 if there is none.
        """
        self.assertIn(n, (2, 3, 4))
        self.assertTrue(1 <= m <= 256)
        h = 1 if n == 2 else 2  # header size for type n-1 block
        # After cancelling the index bytes common to both alternatives,
        # `m` type `n-1` blocks cost `h * m` bytes, while one type `n`
        # block costs `2 + k` bytes.
        #
        #     h * m = 2 + k   ->   k = h * m - 2
        #
        k = h * m - 2  # population where type n-1 and type n tie
        # At the tie (k), the encoder prefers type n-1.
        return max(0, min(255, k - 1))

    def sample_with_highest(self, n, k):
        "Return `k` samples from `range(n)`, with `n-1` first."
        self.assertGreater(n, 0)
        self.assertGreater(k, 0)
        indices = [n - 1]
        indices.extend(sample(range(n - 1), k - 1))
        return indices

    @staticmethod
    def header(nbits):
        "little-endian header using 32-bit struct.pack() for nbits"
        length = bits2bytes(nbits.bit_length())
        b = bytearray([length])
        b.extend(struct.pack("<I", nbits)[:length])
        return bytes(b)

    def make_blob(self, nbits, n, indices):
        "little-endian streams using 32-bit struct.pack() for nbits"
        self.assertIn(n, (2, 3, 4))
        self.assertIn(len(indices), range(256))

        b = bytearray(self.header(nbits))
        b.append(0xc0 + n)
        b.append(len(indices))
        for i in sorted(indices):
            b.extend(struct.pack("<I", i)[:n])
        b.append(0)  # stop byte
        return bytes(b)

    @staticmethod
    def block_counts(n):
        counts = 5 * [0]
        counts[n] = 1
        return counts

    def check_stat(self, a, b, n, check_encode=True):
        stat = sc_stat(b)
        self.assertEqual(stat['nbits'], len(a))
        self.assertEqual(stat['blocks'], self.block_counts(n))
        self.assertEqual(sc_decode(b), a)
        if check_encode:
            self.assertEqual(sc_encode(a), b)


class SC_Tests(unittest.TestCase, Util, SC_Util):

    def test_explicit(self):
        for b, bits, endian in [
                (b'\x00\0',                 '',                  'little'),
                (b'\x01\x03\x01\x03\0',     '110',               'little'),
                (b'\x01\x07\x01\x40\0',     '0000001',           'little'),
                (b'\x11\x07\x01\x02\0',     '0000001',           'big'),
                (b'\x01\x10\x02\xf0\x0f\0', '00001111 11110000', 'little'),
                (b'\x11\x10\xa1\x0c\0',     '00000000 00001000', 'big'),
                (b'\x11\x09\xa1\x08\0',     '00000000 1',        'big'),
                (b'\x01g\xa4abde\0',        97 * '0' + '110110', 'little'),
        ]:
            a = bitarray(bits, endian)
            self.assertEqual(sc_encode(a), b)
            self.assertEQUAL(sc_decode(b), a)

            stat = sc_stat(b)
            self.assertEqual(stat['endian'], endian)
            self.assertEqual(stat['nbits'], len(a))

    def test_skip_block_data(self):
        b = [0x01, 197, 0xa3, None, None, None, 0x00]
        # sc_stat() skips the block data without inspecting its items.
        self.assertEqual(sc_stat(b), {'endian': 'little',
                                      'nbits': 197,
                                      'blocks': [0, 1, 0, 0, 0]})
        # sc_decode() must process the block data.
        self.assertRaises(TypeError, sc_decode, b)

    def test_encode_types(self):
        for a in bitarray('1', 'big'), frozenbitarray('1', 'big'):
            b = sc_encode(a)
            self.assertIs(type(b), bytes)
            self.assertEqual(b, b'\x11\x01\x01\x80\0')

        for a in None, [], 0, 123, b'', b'\x00', 3.14:
            self.assertRaises(TypeError, sc_encode, a)

    def test_decode_stat_types(self):
        blob = b'\x11\x03\x01\x20\0'
        for b in blob, bytearray(blob), list(blob), array.array('B', blob):
            a = sc_decode(b)
            self.assertIs(type(a), bitarray)
            self.assertEqual(a.endian, 'big')
            self.assertEqual(a.to01(), '001')

            stat = sc_stat(b)
            self.assertEqual(stat, {'endian': 'big', 'nbits': 3,
                                    'blocks': [1, 0, 0, 0, 0]})
            self.assertIs(type(stat), dict)
            self.assertIs(type(stat['endian']), str)
            self.assertIs(type(stat['nbits']), int)
            blocks = stat['blocks']
            self.assertIs(type(blocks), list)
            self.assertEqual(len(blocks), 5)
            self.assertTrue(all(type(n) is int for n in blocks))

    def test_zero_explicit(self):
        for blob, blocks, nbits in [
                # empty bitarray
                (b"\x00\0",         [0, 0, 0, 0, 0], 0),
                (b"\x02\x00\x00\xa0\xc2\x00\xc3\x00\xc4\x00\xa0\0",
                                    [0, 2, 1, 1, 1], 0),
                # zero bitarray with length 8
                (b"\x01\x08\0",         [0, 0, 0, 0, 0], 8),
                (b"\x03\x08\x00\x00\0", [0, 0, 0, 0, 0], 8),
                (b"\x01\x08\x01\x00\xa0\xc2\x00\xc3\x00\xc4\x00\xa0\xa0\0",
                                        [1, 3, 1, 1, 1], 8),
        ]:
            it = iter(blob)
            a = sc_decode(it)
            self.assertRaises(StopIteration, next, it)
            self.assertEqual(a.endian, 'little')
            self.assertEqual(len(a), nbits)
            self.assertFalse(a.any())

            it = iter(blob)
            stat = sc_stat(it)
            self.assertRaises(StopIteration, next, it)
            self.assertEqual(stat, {'endian': 'little', 'nbits': nbits,
                                    'blocks': blocks})

        self.assertEqual(sc_encode(bitarray(0, 'little')), b'\x00\0')
        self.assertEqual(sc_encode(bitarray(8, 'little')), b'\x01\x08\0')

    def test_untouch(self):
        blob = b"\x01\x07\x01\x73\0XYZ"

        stream = iter(blob)
        self.assertEqual(sc_decode(stream), bitarray("1100111"))
        self.assertEqual(next(stream), ord('X'))

        stream = iter(blob)
        stat = sc_stat(stream)
        self.assertEqual(stat, {'endian': 'little',
                                'nbits': 7,
                                'blocks': [1, 0, 0, 0, 0]})
        self.assertEqual(next(stream), ord('X'))

    @unittest.skipIf(not HAVE_BINOMIALVARIATE, "Python 3.12+ required")
    def test_concatenate(self):
        N = 100
        arrays = [random_p(randrange(500), 1/8) for _ in range(N)]
        it = iter(b''.join(sc_encode(a) for a in arrays))
        self.assertEqual([sc_decode(it) for _ in range(N)], arrays)
        self.assertRaises(StopIteration, next, it)

    def test_decode_stat_errors(self):
        for f in sc_decode, sc_stat:
            a = [17, 3, 1, 32, 0]

            self.assertRaises(TypeError, f)  # no argument
            self.assertRaises(TypeError, f, a, None)  # extra argument
            f(a)  # no error
            for x in 256, -1:
                a[-1] = x
                self.assertRaises(ValueError, f, a)

            a = [0x02, None]
            self.assertRaises(TypeError, f, a)
            for x in None, 3, 3.2, Ellipsis, 'foo':
                self.assertRaises(TypeError, f, x)

    def test_decode_header_errors(self):
        for f in sc_decode, sc_stat:
            # invalid header
            for c in 0x20, 0x21, 0x40, 0x80, 0xc0, 0xf0, 0xff:
                self.assertRaisesMessage(ValueError,
                                         "invalid header: 0x%02x" % c,
                                         f, [c])
            # invalid block head
            for c in 0xc0, 0xc1, 0xc5, 0xff:
                self.assertRaisesMessage(ValueError,
                                         "invalid block head: 0x%02x" % c,
                                         f, [0x01, 0x10, c])

    def test_decode_header_overflow(self):
        self.assertRaisesMessage(
            OverflowError,
            "sizeof(Py_ssize_t) = %d: cannot read 9 bytes" % PTRSIZE,
            sc_decode, b'\x09' + 9 * b'\x00')

        self.assertRaisesMessage(
            ValueError,
            "read %d bytes got negative value: -1" % PTRSIZE,
            sc_decode, [PTRSIZE] + PTRSIZE * [0xff])

        if PTRSIZE == 4:
            self.assertRaisesMessage(
                OverflowError,
                "sizeof(Py_ssize_t) = 4: cannot read 5 bytes",
                sc_decode, b'\x05' + 5 * b'\x00')

            self.assertRaisesMessage(
                ValueError,
                "read 4 bytes got negative value: -2147483648",
                sc_decode, b'\x04\x00\x00\x00\x80')

    def test_decode_errors(self):
        # too many raw bytes
        self.assertRaisesMessage(
            ValueError, "decode error (raw): 0 + 2 > 1",
            sc_decode, b"\x01\x05\x02\xff\xff\0")
        self.assertRaisesMessage(
            ValueError, "decode error (raw): 32 + 3 > 34",
            sc_decode, b"\x02\x0f\x01\xa0\x03\xff\xff\xff\0")

        # sparse index too high
        self.assertRaisesMessage(
            ValueError, "decode error (n=1): 128 >= 128",
            sc_decode, b"\x01\x80\xa1\x80\0")
        self.assertRaisesMessage(
            ValueError, "decode error (n=2): 512 >= 512",
            sc_decode, b"\x02\x00\x02\xc2\x01\x00\x02\0")
        self.assertRaisesMessage(
            ValueError, "decode error (n=3): 32768 >= 32768",
            sc_decode, b"\x02\x00\x80\xc3\x01\x00\x80\x00\0")

        msg = {4: "read 4 bytes got negative value: -2147483648",
               8: "decode error (n=4): 2147483648 >= 16"}
        self.assertRaisesMessage(
            ValueError, msg[PTRSIZE],
            sc_decode, b"\x01\x10\xc4\x01\x00\x00\x00\x80\0")

        msg = {4: "read 4 bytes got negative value: -1",
               8: "decode error (n=4): 4294967295 >= 16"}
        self.assertRaisesMessage(
            ValueError, msg[PTRSIZE],
            sc_decode, b"\x01\x10\xc4\x01\xff\xff\xff\xff\0")

    def test_stop_iteration(self):
        for stream in [b'', b'\x00', b'\x01', b'\x02\x77',
                       b'\x01\x04\x01', b'\x01\x04\xa1', b'\x01\x04\xa0']:
            for f in sc_decode, sc_stat:
                self.assertRaises(StopIteration, f, stream)

    def test_decode_ambiguity(self):
        for n, blob in [
                # raw:
                (0, b'\x11\x03\x01\x20\0'), # this is what sc_encode gives us
                (0, b'\x11\x03\x01\x3f\0'), # set the pad bits to 1
                (0, b'\x12\x03\x00\x01\x20\0'), # header has redundant zeros
                # sparse:
                (1, b'\x11\x03\xa1\x02\0'),
                (2, b'\x11\x03\xc2\x01\x02\x00\0'),
                (3, b'\x11\x03\xc3\x01\x02\x00\x00\0'),
                (4, b'\x11\x03\xc4\x01\x02\x00\x00\x00\0'),
        ]:
            a = sc_decode(blob)
            self.assertEqual(a.to01(), '001')
            stat = sc_stat(blob)
            self.assertEqual(stat['endian'], 'big')
            self.assertEqual(stat['nbits'], 3)
            self.assertEqual(stat['blocks'], self.block_counts(n))

    def test_block_type0(self):
        for k in range(0x01, 0xa0):
            nbytes = k if k <= 32 else 32 * (k - 31)
            nbits = 8 * nbytes
            a = ones(nbits, "little")
            b = bytearray(self.header(nbits))
            b.append(k)
            b.extend(a.tobytes())
            b.append(0)  # stop byte

            self.check_stat(a, b, 0)

    def test_block_type1(self):
        a = bitarray(256, 'little')
        indices = sample(range(256), 31)

        for k, index in enumerate(indices, 1):
            a[index] = 1
            b = bytearray([0x02, 0x00, 0x01, 0xa0 + k])
            b.extend(sorted(indices[:k]))
            b.append(0)  # stop byte

            self.check_stat(a, b, 1)

    def test_block_type234(self):
        # nbits and k are redundant, but make the expected values explicit;
        # k also tests get_max_pop().
        for n, m, nbits, k in [
                (2, 256, 1 << 16, 253),
                (3, 256, 1 << 24, 255),
                (4,   4, 1 << 26,   5),
        ]:
            prev_size = 1 << ((n - 1) * 8)  # size of type n-1 block
            self.assertEqual(prev_size * m, nbits)
            a = bitarray(nbits, 'little')
            self.assertEqual(self.get_max_pop(n, m), k)
            indices = self.sample_with_highest(nbits, k)
            a[indices] = 1
            b = self.make_blob(nbits, n, indices)
            self.check_stat(a, b, n)

    def test_sparse_population_limit(self):
        a = zeros(1 << 24)
        a[::1 << 16] = 1  # 256 populated type-2 blocks
        b = sc_encode(a)
        self.assertEqual(sc_decode(b), a)
        # The index count byte can represent at most 255 indices (0xff), so
        # the 256 indices do not fit in one type-3 block.  The encoder first
        # uses one type-2 block, leaving 255 indices which fit in one type-3
        # block.
        self.assertEqual(sc_stat(b)['blocks'], [0, 0, 1, 1, 0])

    def test_decode_random_bytes(self):
        # ensure random input doesn't crash the decoder
        for _ in range(100):
            n = randrange(20)
            b = b'\x02\x00\x04' + os.urandom(n)
            try:
                a = sc_decode(b)
            except (StopIteration, ValueError):
                continue
            self.assertEqual(len(a), 1024)
            self.assertEqual(a.endian, 'little')

    def check_blob_length(self, a, m, blocks):
        blob = sc_encode(a)
        self.assertEqual(len(blob), m)
        self.assertEqual(sc_decode(blob), a)
        stat = sc_stat(blob)
        self.assertEqual(stat['nbits'], len(a))
        self.assertEqual(stat['blocks'], blocks)

    def test_encode_zeros(self):
        for i in range(26):
            nbits = 1 << i
            a = zeros(nbits)
            m = 2                                # head byte and stop byte
            m += bits2bytes(nbits.bit_length())  # size of nbits in bytes
            self.check_blob_length(a, m, [0, 0, 0, 0, 0])

            a[0] = 1
            # As the first block accounts for the entire population, trailing
            # zeros do not cause the block type to increase with array size.
            n = int(i > 3)
            m += 2  # block head and raw byte (type 0) or index (type 1)

            self.check_blob_length(a, m, self.block_counts(n))

    def test_encode_trailing_zeros(self):
        for n, index, blocks in [
                (1, 0,       self.block_counts(1)),
                (2, 1 << 8,  [0, 2, 0, 0, 0]),  # 2 type 1 blocks are smaller
                (2, 3 << 8,  self.block_counts(2)),
                (3, 1 << 16, self.block_counts(3)),
        ]:
            # The bitarray is 4 times the decoded size of a type n block.
            a = bitarray(4 << (8 * n))
            a[index] = 1
            blob = sc_encode(a)
            # Trailing zeros must not widen the encoding.
            self.assertEqual(sc_stat(blob)['blocks'], blocks)
            self.assertEqual(sc_decode(blob), a)

    def test_encode_ones(self):
        for _ in range(10):
            nbits = randrange(100_000)
            a = ones(nbits)
            m = 2                                # head byte and stop byte
            m += bits2bytes(nbits.bit_length())  # size bytes
            nbytes = bits2bytes(nbits)
            m += nbytes                          # actual raw bytes
            # number of raw blocks:
            raw_blocks = (
                bool(nbytes % 32) +              # number in 0x01 .. 0x1f
                (nbytes // 32 + 127) // 128)     # number in 0x20 .. 0x9f
            # number of head bytes
            m += raw_blocks
            self.check_blob_length(a, m, [raw_blocks, 0, 0, 0, 0])

    def round_trip(self, a):
        c = a.copy()
        blob = sc_encode(a)
        b = sc_decode(blob)
        self.assertTrue(a == b == c)
        self.assertTrue(a.endian == b.endian == c.endian)
        stat = sc_stat(blob)
        self.assertEqual(stat['endian'], a.endian)
        self.assertEqual(stat['nbits'], len(a))

    def test_random(self):
        for _ in range(5):
            n = randrange(100_000)
            endian = choice(ENDIANS)
            a = ones(n, endian)
            while a.any():
                a &= urandom(n, endian)
                self.round_trip(a)


# ------------------------   variable length codec   ------------------------

class VL_Tests(unittest.TestCase, Util):

    def test_explicit(self):
        for blob, s in [
                (b'\x40', ''),
                (b'\x30', '0'),
                (b'\x38', '1'),
                (b'\x00', '0000'),
                (b'\x01', '0001'),
                (b'\xd3\x20', '001101'),
                (b'\xe0\x40', '0000 1'),
                (b'\x90\x02', '0000 000001'),
                (b'\xb5\xa7\x18', '0101 0100111 0011'),
                (b'\x95\xb7\x1c', '0101 0110111 001110'),
        ]:
            a = bitarray(s)
            self.assertEqual(vl_encode(a), blob)
            c = vl_decode(blob)
            self.assertEqual(c, a)
            self.assertEqual(c.endian, get_default_endian())

            for endian in OPT_ENDIANS:
                a = bitarray(s, endian)
                c = vl_encode(a)
                self.assertIs(type(c), bytes)
                self.assertEqual(c, blob)

                c = vl_decode(blob, endian)
                self.assertEqual(c, a)
                self.assertEqual(c.endian, endian or get_default_endian())

    def test_encode_types(self):
        s = "0011 01"
        for a in bitarray(s), frozenbitarray(s):
            b = vl_encode(a)
            self.assertIs(type(b), bytes)
            self.assertEqual(b, b'\xd3\x20')

        for a in None, [], 0, 123, b'', b'\x00', 3.14:
            self.assertRaises(TypeError, vl_encode, a)

    def test_decode_types(self):
        blob = b'\xd3\x20'
        for s in (blob, iter(blob), memoryview(blob), iter([0xd3, 0x20]),
                  bytearray(blob)):
            a = vl_decode(s, endian=choice(ENDIANS))
            self.assertIs(type(a), bitarray)
            self.assertEqual(a, bitarray('0011 01'))

        # these objects are not iterable
        for arg in None, 0, 1, 0.0:
            self.assertRaises(TypeError, vl_decode, arg)
        # these items cannot be interpreted as ints
        for item in None, 2.34, Ellipsis, 'foo':
            self.assertRaises(TypeError, vl_decode, iter([0x95, item]))

    def test_decode_args(self):
        # item not integer
        self.assertRaises(TypeError, vl_decode, iter([b'\x40']))

        self.assertRaises(TypeError, vl_decode, b'\x40', 'big', 3)
        self.assertRaises(ValueError, vl_decode, b'\x40', 'foo')

    def test_decode_trailing(self):
        for s, bits in [(b'\x40ABC', ''),
                        (b'\xe0\x40A', '00001')]:
            stream = iter(s)
            self.assertEqual(vl_decode(stream), bitarray(bits))
            self.assertEqual(next(stream), 65)

    def test_decode_ambiguity(self):
        for s in b'\x40', b'\x4f', b'\x45':
            self.assertEqual(vl_decode(s), bitarray())
        for s in b'\x1e', b'\x1f':
            self.assertEqual(vl_decode(s), bitarray('111'))

    def test_decode_stream(self):
        stream = iter(b'\x40\x30\x38\x40\x2c\xe0\x40\xd3\x20')
        for bits in '', '0', '1', '', '11', '0000 1', '0011 01':
            self.assertEqual(vl_decode(stream), bitarray(bits))

        arrays = [urandom(randrange(30)) for _ in range(1000)]
        stream = iter(b''.join(vl_encode(a) for a in arrays))
        for a in arrays:
            self.assertEqual(vl_decode(stream), a)

    def test_decode_errors(self):
        # decode empty bytes
        self.assertRaises(StopIteration, vl_decode, b'')
        # invalid head byte
        for s in [
                b'\x70', b'\xf0',           # padding = 7
                b'\x50', b'\x60', b'\x70',  # no second byte, but padding > 4
        ]:
            self.assertRaisesMessage(ValueError,
                                     "invalid head byte: 0x%02x" % s[0],
                                     vl_decode, s)
        # high bit set, but no terminating byte
        for s in b'\x80', b'\x80\x80':
            self.assertRaises(StopIteration, vl_decode, s)
        # decode list with out of range items
        for i in -1, 256:
            self.assertRaises(ValueError, vl_decode, [i])
        # wrong type
        self.assertRaises(TypeError, vl_decode, [None])

    def test_decode_invalid_stream(self):
        N = 100
        s = iter(N * (3 * [0x80] + ['XX']) + ['end.'])
        for _ in range(N):
            a = None
            try:
                a = vl_decode(s)
            except TypeError:
                pass
            self.assertIsNone(a)
        self.assertEqual(next(s), 'end.')

    def test_explicit_zeros(self):
        for n in range(100):
            a = zeros(4 + n * 7)
            s = n * b'\x80' + b'\x00'
            self.assertEqual(vl_encode(a), s)
            self.assertEqual(vl_decode(s), a)

    @unittest.skipIf(PTRSIZE != 8, 'requires 64 bit machine')
    def test_uleb128(self):
        LEN_PAD_BITS = 3
        for n in range(1, 61):
            a = urandom(n, 'little')
            a[-1] = 1  # avoid overlong ULEB128 representation
            b = vl_encode(a)

            # Treat the VL bytes as canonical ULEB128.
            payload = bitarray(b, endian='little')
            del payload[7::8]  # remove each byte's continuation bit
            i = ba2int(payload)
            self.assertEqual(uleb128_encode(i), b)

            # Remove continuation bits, padding metadata, and actual padding
            # to recover the original bitarray.
            c = bitarray(b, 'big')
            del c[::8]  # remove each byte's continuation bit
            padding = ba2int(c[:LEN_PAD_BITS])
            self.assertEqual(LEN_PAD_BITS + n + padding, 7 * len(b))
            del c[:LEN_PAD_BITS]    # remove padding metadata
            del c[len(c) - padding:]  # remove actual padding
            self.assertEqual(c, a)

    def test_concatenate(self):
        N = 100
        arrays = [urandom(randrange(20)) for _ in range(N)]
        it = iter(b''.join(vl_encode(a) for a in arrays))
        self.assertEqual([vl_decode(it) for _ in range(N)], arrays)
        self.assertRaises(StopIteration, next, it)

    def round_trip(self, a):
        c = a.copy()
        s = vl_encode(a)
        b = vl_decode(s)
        self.check_obj(b)
        self.assertTrue(a == b == c)
        LEN_PAD_BITS = 3
        HEAD_WIDTH = 7 - LEN_PAD_BITS  # number of payload bits in head
        self.assertEqual(len(s), (len(a) + LEN_PAD_BITS + 6) // 7)
        head = s[0]
        padding = (head & 0x70) >> HEAD_WIDTH
        self.assertEqual(LEN_PAD_BITS + len(a) + padding, 7 * len(s))

        # overflow-safe C implementation in _util.c
        r = len(a) % 7
        self.assertEqual(len(a) // 7 + 1 + (r > HEAD_WIDTH), len(s))
        self.assertEqual((HEAD_WIDTH - r + 7) % 7, padding)

    def test_large(self):
        for _ in range(5):
            a = urandom(randrange(100_000))
            self.round_trip(a)

    def test_random(self):
        for n in range(50):
            a = urandom_2(n)
            self.round_trip(a)

# ---------------------------------------------------------------------------

class IntegerizationTests(unittest.TestCase, Util):

    def test_ba2int(self):
        self.assertEqual(ba2int(bitarray('0')), 0)
        self.assertEqual(ba2int(bitarray('1')), 1)
        self.assertEqual(ba2int(bitarray('00101', 'big')), 5)
        self.assertEqual(ba2int(bitarray('00101', 'little')), 20)
        self.assertEqual(ba2int(frozenbitarray('11')), 3)
        self.assertRaises(ValueError, ba2int, bitarray())
        self.assertRaises(ValueError, ba2int, frozenbitarray())
        self.assertRaises(TypeError, ba2int, '101')
        a = bitarray('111')
        b = a.copy()
        self.assertEqual(ba2int(a), 7)
        # ensure original object wasn't altered
        self.assertEQUAL(a, b)

    def test_ba2int_frozen(self):
        for a in self.randombitarrays(start=1):
            b = frozenbitarray(a)
            self.assertEqual(ba2int(b), ba2int(a))
            self.assertEQUAL(a, b)

    def test_ba2int_random(self):
        for a in self.randombitarrays(start=1):
            b = bitarray(a, 'big')
            self.assertEqual(a, b)
            self.assertEqual(ba2int(b), int(b.to01(), 2))

    def test_ba2int_bytes(self):
        for n in range(1, 50):
            a = urandom_2(8 * n)
            c = bytearray(a.tobytes())
            i = 0
            for x in (c if a.endian == 'big' else reversed(c)):
                i <<= 8
                i |= x
            self.assertEqual(ba2int(a), i)

    def test_int2ba(self):
        self.assertEqual(int2ba(0), bitarray('0'))
        self.assertEqual(int2ba(1), bitarray('1'))
        self.assertEqual(int2ba(5), bitarray('101'))
        self.assertEQUAL(int2ba(6, endian='big'), bitarray('110', 'big'))
        self.assertEQUAL(int2ba(6, endian='little'),
                         bitarray('011', 'little'))

    def test_errors(self):
        self.assertRaises(TypeError, int2ba, 1.0)
        self.assertRaises(TypeError, int2ba, 1, 3.0)
        self.assertRaises(ValueError, int2ba, 1, 0)
        self.assertRaises(TypeError, int2ba, 1, 10, 123)
        self.assertRaises(ValueError, int2ba, 1, 10, 'asd')
        # signed integer requires length
        self.assertRaises(TypeError, int2ba, 100, signed=True)
        # argument cannot be float
        self.assertRaises(TypeError, int2ba, 123.0)

    def test_signed(self):
        for s, i in [
                ('0',  0),
                ('1', -1),
                ('00',  0),
                ('10',  1),
                ('01', -2),
                ('11', -1),
                ('000',  0),
                ('100',  1),
                ('010',  2),
                ('110',  3),
                ('001', -4),
                ('101', -3),
                ('011', -2),
                ('111', -1),
                ('00000',   0),
                ('11110',  15),
                ('00001', -16),
                ('11111',  -1),
                ('00000000 0',    0),
                ('11111111 0',  255),
                ('00000000 1', -256),
                ('11111111 1',   -1),
        ]:
            self.assertEqual(ba2int(bitarray(s, 'little'), signed=1), i)
            self.assertEqual(ba2int(bitarray(s[::-1], 'big'), signed=1), i)

            len_s = len(bitarray(s))
            self.assertEQUAL(int2ba(i, len_s, 'little', signed=1),
                             bitarray(s, 'little'))
            self.assertEQUAL(int2ba(i, len_s, 'big', signed=1),
                             bitarray(s[::-1], 'big'))

    def test_zero(self):
        for endian in "little", "big":
            a = int2ba(0, endian=endian)
            self.assertEQUAL(a, bitarray('0', endian=endian))
            for n in range(1, 100):
                a = int2ba(0, length=n, endian=endian, signed=True)
                b = bitarray(n * '0', endian)
                self.assertEQUAL(a, b)
                for signed in 0, 1:
                    self.assertEqual(ba2int(b, signed=signed), 0)

    def test_negative_one(self):
        for endian in "little", "big":
            for n in range(1, 100):
                a = int2ba(-1, length=n, endian=endian, signed=True)
                b = bitarray(n * '1', endian)
                self.assertEQUAL(a, b)
                self.assertEqual(ba2int(b, signed=True), -1)

    def test_int2ba_overflow(self):
        self.assertRaises(OverflowError, int2ba, -1)
        self.assertRaises(OverflowError, int2ba, -1, 4)

        self.assertRaises(OverflowError, int2ba, 128, 7)
        self.assertRaises(OverflowError, int2ba, 64, 7, signed=1)
        self.assertRaises(OverflowError, int2ba, -65, 7, signed=1)

        for n in range(1, 20):
            self.assertRaises(OverflowError, int2ba, 1 << n, n)
            self.assertRaises(OverflowError, int2ba, 1 << (n - 1), n,
                              signed=1)
            self.assertRaises(OverflowError, int2ba, -(1 << (n - 1)) - 1, n,
                              signed=1)

    def test_int2ba_length(self):
        self.assertRaises(TypeError, int2ba, 0, 1.0)
        self.assertRaises(ValueError, int2ba, 0, 0)
        self.assertEqual(int2ba(5, length=6, endian='big'),
                         bitarray('000101'))
        for n in range(1, 100):
            ab = int2ba(1, n, 'big')
            al = int2ba(1, n, 'little')
            self.assertEqual(ab.endian, 'big')
            self.assertEqual(al.endian, 'little')
            self.assertEqual(len(ab), n),
            self.assertEqual(len(al), n)
            self.assertEqual(ab, bitarray((n - 1) * '0') + bitarray('1'))
            self.assertEqual(al, bitarray('1') + bitarray((n - 1) * '0'))

            ab = int2ba(0, n, 'big')
            al = int2ba(0, n, 'little')
            self.assertEqual(len(ab), n)
            self.assertEqual(len(al), n)
            self.assertEqual(ab, bitarray(n * '0', 'big'))
            self.assertEqual(al, bitarray(n * '0', 'little'))

            self.assertEqual(int2ba(2 ** n - 1), bitarray(n * '1'))
            self.assertEqual(int2ba(2 ** n - 1, endian='little'),
                             bitarray(n * '1'))

    def test_explicit(self):
        for i, sa in [( 0,     '0'),    (1,         '1'),
                      ( 2,    '10'),    (3,        '11'),
                      (25, '11001'),  (265, '100001001'),
                      (3691038, '1110000101001000011110')]:
            ab = bitarray(sa, 'big')
            al = bitarray(sa[::-1], 'little')
            self.assertEQUAL(int2ba(i),
                             ab if get_default_endian() == 'big' else al)
            self.assertEQUAL(int2ba(i, endian='big'), ab)
            self.assertEQUAL(int2ba(i, endian='little'), al)
            self.assertEqual(ba2int(ab), ba2int(al), i)

    def check_round_trip(self, i):
        for endian in 'big', 'little':
            a = int2ba(i, endian=endian)
            self.check_obj(a)
            self.assertEqual(a.endian, endian)
            self.assertTrue(len(a) > 0)
            # ensure we have no leading zeros
            if a.endian == 'big':
                self.assertTrue(len(a) == 1 or a.index(1) == 0)
            self.assertEqual(ba2int(a), i)
            if i > 0:
                self.assertEqual(i.bit_length(), len(a))
            # add a few trailing / leading zeros to bitarray
            if endian == 'big':
                a = zeros(randrange(4), endian) + a
            else:
                a = a + zeros(randrange(4), endian)
            self.assertEqual(a.endian, endian)
            self.assertEqual(ba2int(a), i)

    def test_many(self):
        for _ in range(20):
            self.check_round_trip(randrange(10 ** randint(3, 300)))

    @staticmethod
    def twos_complement(i, num_bits):
        # https://en.wikipedia.org/wiki/Two%27s_complement
        mask = 2 ** (num_bits - 1)
        return -(i & mask) + (i & ~mask)

    def test_random_signed(self):
        for a in self.randombitarrays(start=1):
            i = ba2int(a, signed=True)
            b = int2ba(i, len(a), a.endian, signed=True)
            self.assertEQUAL(a, b)

            j = ba2int(a, signed=False)  # unsigned
            if i >= 0:
                self.assertEqual(i, j)

            self.assertEqual(i, self.twos_complement(j, len(a)))

# ---------------------------------------------------------------------------

class MixedTests(unittest.TestCase, Util):

    def test_bin(self):
        for _ in range(20):
            i = randrange(1000)
            s = bin(i)
            self.assertEqual(s[:2], '0b')
            a = bitarray(s[2:], 'big')
            self.assertEqual(ba2int(a), i)
            t = a.to01()
            self.assertEqual(t, s[2:])
            self.assertEqual(int(t, 2), i)

    def test_oct(self):
        for _ in range(20):
            i = randrange(1000)
            s = oct(i)
            self.assertEqual(s[:2], '0o')
            a = base2ba(8, s[2:], 'big')
            self.assertEqual(ba2int(a), i)
            t = ba2base(8, a)
            self.assertEqual(t, s[2:])
            self.assertEqual(int(t, 8), i)

    def test_hex(self):
        for _ in range(20):
            i = randrange(1000)
            s = hex(i)
            self.assertEqual(s[:2], '0x')
            a = hex2ba(s[2:], 'big')
            self.assertEqual(ba2int(a), i)
            t = ba2hex(a)
            self.assertEqual(t, s[2:])
            self.assertEqual(int(t, 16), i)

    def test_bitwise(self):
        for a in self.randombitarrays(start=1):
            b = urandom(len(a), a.endian)
            aa = a.copy()
            bb = b.copy()
            i = ba2int(a)
            j = ba2int(b)
            self.assertEqual(ba2int(a & b), i & j)
            self.assertEqual(ba2int(a | b), i | j)
            self.assertEqual(ba2int(a ^ b), i ^ j)

            n = randint(0, len(a))
            if a.endian == 'big':
                self.assertEqual(ba2int(a >> n), i >> n)
                c = zeros(len(a), 'big') + a
                self.assertEqual(ba2int(c << n), i << n)

            self.assertEQUAL(a, aa)
            self.assertEQUAL(b, bb)

    def test_bitwise_inplace(self):
        for a in self.randombitarrays(start=1):
            b = urandom(len(a), a.endian)
            bb = b.copy()
            i = ba2int(a)
            j = ba2int(b)
            c = a.copy()
            c &= b
            self.assertEqual(ba2int(c), i & j)
            c = a.copy()
            c |= b
            self.assertEqual(ba2int(c), i | j)
            c = a.copy()
            c ^= b
            self.assertEqual(ba2int(c), i ^ j)
            self.assertEQUAL(b, bb)

            n = randint(0, len(a))
            if a.endian == 'big':
                c = a.copy()
                c >>= n
                self.assertEqual(ba2int(c), i >> n)
                c = zeros(len(a), 'big') + a
                c <<= n
                self.assertEqual(ba2int(c), i << n)

# ----------------------  serialize()  deserialize()  -----------------------

class SerializationTests(unittest.TestCase, Util):

    def test_explicit(self):
        for blob, endian, bits in [
                (b'\x00',         'little', ''),
                (b'\x07\x01',     'little', '1'),
                (b'\x17\x80',     'big',    '1'),
                (b'\x13\xf8',     'big',    '11111'),
                (b'\x00\x0f',     'little', '11110000'),
                (b'\x10\xf0',     'big',    '11110000'),
                (b'\x12\x87\xd8', 'big',    '10000111 110110')
        ]:
            a = bitarray(bits, endian)
            s = serialize(a)
            self.assertEqual(blob, s)
            self.assertIs(type(s), bytes)

            b = deserialize(blob)
            self.assertEqual(b, a)
            self.assertEqual(b.endian, endian)
            self.assertIs(type(b), bitarray)

    def test_serialize_args(self):
        for x in '0', 0, 1, b'\x00', 0.0, [0, 1], bytearray([0]):
            self.assertRaises(TypeError, serialize, x)
        # no arguments
        self.assertRaises(TypeError, serialize)
        # too many arguments
        self.assertRaises(TypeError, serialize, bitarray(), 1)

        for a in bitarray('0111', 'big'), frozenbitarray('0111', 'big'):
            self.assertEqual(serialize(a), b'\x14\x70')

    def test_deserialize_args(self):
        for x in 0, 1, False, True, None, '', '01', 0.0, [0, 1]:
            self.assertRaises(TypeError, deserialize, x)
        # no arguments
        self.assertRaises(TypeError, deserialize)
        # too many arguments
        self.assertRaises(TypeError, deserialize, b'\x00', 1)

        blob = b'\x03\x06'
        x = bitarray(blob)
        for s in blob, bytearray(blob), memoryview(blob), x:
            a = deserialize(s)
            self.assertEqual(a.to01(), '01100')
            self.assertEqual(a.endian, 'little')

    def test_invalid_bytes(self):
        self.assertRaises(ValueError, deserialize, b'')

        def check_msg(b):
            msg = "invalid header byte: 0x%02x" % b[0]
            self.assertRaisesMessage(ValueError, msg, deserialize, b)

        for i in range(256):
            b = bytearray([i])
            if i == 0 or i == 16:
                self.assertEqual(deserialize(b), bitarray())
            else:
                self.assertRaises(ValueError, deserialize, b)
                check_msg(b)

            b.append(0)
            if i < 32 and i % 16 < 8:
                self.assertEqual(deserialize(b), zeros(8 - i % 8))
            else:
                self.assertRaises(ValueError, deserialize, b)
                check_msg(b)

    def test_padbits_ignored(self):
        for blob, endian in [
                (b'\x07\x01', 'little'),
                (b'\x07\x03', 'little'),
                (b'\x07\xff', 'little'),
                (b'\x17\x80', 'big'),
                (b'\x17\xc0', 'big'),
                (b'\x17\xff', 'big'),
        ]:
            a = deserialize(blob)
            self.assertEqual(a.to01(), '1')
            self.assertEqual(a.endian, endian)

    def test_random(self):
        for a in self.randombitarrays():
            b = serialize(a)
            c = deserialize(b)
            self.assertEqual(a, c)
            self.assertEqual(a.endian, c.endian)
            self.check_obj(c)

# ------------------------------  Huffman coding  ---------------------------

class HuffmanUtil:

    def check_code(self, code):
        self.assertIs(type(code), dict)
        for v in code.values():
            self.assertIs(type(v), frozenbitarray)
            self.assertGreater(len(v), 0)

        if sys.version_info[:2] >= (3, 15):
            hash(frozendict(code))

        tree = decodetree(code)
        self.assertEqual(tree.todict(), code)
        n = len(code)
        nodes = tree.nodes()
        if n == 1:
            # one-symbol code is incomplete
            self.assertEqual(nodes, (1, 0, 1))
        else:
            # proper Huffman tree is complete
            self.assertEqual(nodes, (0, n - 1, n))


class CommonHuffmanTests(HuffmanUtil, unittest.TestCase):

    # tests for both huffman_code() and canonical_huffman()

    def test_wrong_arg(self):
        for F in huffman_code, canonical_huffman:
            self.assertRaises(TypeError, F)
            self.assertRaises(TypeError, F, [('a', 1)])
            self.assertRaises(TypeError, F, 123)
            self.assertRaises(TypeError, F, None)
            # cannot compare 'a' with 1
            self.assertRaises(TypeError, F, {'A': 'a', 'B': 1})
            # frequency map cannot be empty
            d = {}
            self.assertRaises(ValueError, F, d)
            self.assertEqual(d, {})  # d wasn't changed by F
            # wrong endianness type / extra argument
            self.assertRaises(TypeError, F, {'A': 2, 'B': 1}, 1)

    @staticmethod
    def create_codes(freq):
        yield huffman_code(freq)
        yield canonical_huffman(freq)[0]

    def test_small_range(self):
        for n in range(1, 10):
            f = {i: randrange(10) for i in range(n)}
            f2 = dict(f)
            for code in self.create_codes(f):
                self.assertEqual(len(code), n)
                self.check_code(code)
            # make sure frequency map wasn't changed
            self.assertEqual(f, f2)

    @unittest.skipIf(sys.version_info[:2] < (3, 15),
                     "frozendict introduced in Python 3.15")
    def test_from_frozendict(self):
        for n in range(1, 10):
            f = frozendict({i: random() for i in range(n)})
            for code in self.create_codes(f):
                self.assertEqual(len(code), n)
                self.check_code(code)

    def test_from_counter(self):
        c = Counter("abracadabra")
        for code in self.create_codes(c):
            self.check_code(code)

    def test_large(self):
        N = randrange(1000, 2000)
        f = {i: random() for i in range(N)}
        for code in self.create_codes(f):
            self.assertEqual(len(code), N)
            self.check_code(code)

    def test_balanced(self):
        n = 7
        freq = {}
        for i in range(1 << n):
            freq[i] = 1
        for code in self.create_codes(freq):
            self.assertEqual(len(code), 1 << n)
            for v in code.values():
                self.assertEqual(len(v), n)
            self.check_code(code)

    def test_unbalanced(self):
        n = 37
        freq = {}
        for i in range(n):
            freq[i] = 1 << i
        for code in self.create_codes(freq):
            self.assertEqual(len(code), n)
            for i in range(n):
                self.assertEqual(len(code[i]), n - max(1, i))
            self.check_code(code)

    def test_random_list(self):
        plain = choices(range(100), k=500)
        for code in self.create_codes(Counter(plain)):
            a = bitarray()
            a.encode(code, plain)
            self.assertEqual(list(a.decode(code)), plain)
            self.check_code(code)


class HuffmanTests(HuffmanUtil, unittest.TestCase):

    def test_simple(self):
        freq = {0: 10, 'as': 2, None: 1.6}
        code = huffman_code(freq)
        self.assertEqual(len(code), 3)
        self.assertEqual(len(code[0]), 1)
        self.assertEqual(len(code['as']), 2)
        self.assertEqual(len(code[None]), 2)

    def test_endianness_and_type(self):
        freq = {'A': 10, 'B': 2, 'C': 5}
        for endian in 'big', 'little':
            code = huffman_code(freq, endian)
            self.assertEqual(len(code), 3)
            for v in code.values():
                self.assertEqual(v.endian, endian)
                self.assertIs(type(v), frozenbitarray)

    def test_one_symbol(self):
        cnt = {'a': 1}
        endian = choice(ENDIANS)
        code = huffman_code(cnt, endian)
        self.assertEqual(code, {'a': bitarray('0')})
        v = code['a']
        self.assertEqual(v.endian, endian)
        self.assertIs(type(v), frozenbitarray)
        for n in range(4):
            msg = n * ['a']
            a = bitarray()
            a.encode(code, msg)
            self.assertEqual(a, zeros(n))
            self.assertEqual(list(a.decode(code)), msg)
            a.append(1)
            self.assertRaises(ValueError, list, a.decode(code))

    def test_counter(self):
        message = 'the quick brown fox jumps over the lazy dog.'
        code = huffman_code(Counter(message))
        a = bitarray()
        a.encode(code, message)
        self.assertEqual(''.join(a.decode(code)), message)
        self.check_code(code)


class CanonicalHuffmanTests(Util, HuffmanUtil, unittest.TestCase):

    def test_basic(self):
        plain = bytearray(b'the quick brown fox jumps over the lazy dog.')
        chc, count, symbol = canonical_huffman(Counter(plain))
        self.assertIs(type(chc), dict)
        for v in chc.values():
            self.assertIs(type(v), frozenbitarray)
        self.assertIs(type(count), list)
        self.assertIs(type(symbol), list)
        a = bitarray()
        a.encode(chc, plain)
        self.assertEqual(bytearray(a.decode(chc)), plain)
        self.assertEqual(bytearray(canonical_decode(a, count, symbol)), plain)

    def test_example(self):
        F = frozenbitarray
        cnt = {'a': 5, 'b': 3, 'c': 1, 'd': 1, 'r': 2}
        codedict, count, symbol = canonical_huffman(cnt)
        self.assertEqual(codedict, {'a': F('0'),
                                    'b': F('10'),
                                    'c': F('1110'),
                                    'd': F('1111'),
                                    'r': F('110')})
        self.assertEqual(count, [0, 1, 1, 1, 2])
        self.assertEqual(symbol, ['a', 'b', 'r', 'c', 'd'])
        a = bitarray('01011001110011110101100')
        msg = "abracadabra"
        self.assertEqual(''.join(a.decode(codedict)), msg)
        self.assertEqual(''.join(canonical_decode(a, count, symbol)), msg)

    def test_one_symbol(self):
        cnt = {'a': 1}
        chc, count, symbol = canonical_huffman(cnt)
        self.assertEqual(chc, {'a': bitarray('0')})
        self.assertIs(type(chc['a']), frozenbitarray)
        self.assertEqual(count, [0, 1])
        self.assertEqual(symbol, ['a'])
        for n in range(4):
            msg = n * ['a']
            a = bitarray()
            a.encode(chc, msg)
            self.assertEqual(a.to01(), n * '0')
            self.assertEqual(list(canonical_decode(a, count, symbol)), msg)
            a.append(1)
            self.assertRaises(ValueError, list,
                              canonical_decode(a, count, symbol))

    def test_canonical_decode_errors(self):
        C = canonical_decode
        a = bitarray('1101')
        s = ['a']
        # bitarray not of bitarray type
        self.assertRaises(TypeError, C, '11', [0, 1], s)
        # count not sequence
        self.assertRaises(TypeError, C, a, {0, 1}, s)
        # count element not an int
        self.assertRaises(TypeError, C, a, [0, 1.0], s)
        # count element overflow
        self.assertRaises(OverflowError, C, a, [0, 1 << 65], s)
        # symbol not sequence
        self.assertRaises(TypeError, C, a, [0, 1], 43)

        symbol = ['a', 'b', 'c', 'd']
        # sum(count) != len(symbol)
        self.assertRaisesMessage(ValueError,
                                 "sum(count) = 3, but len(symbol) = 4",
                                 C, a, [0, 1, 2], symbol)
        # count list too long
        self.assertRaisesMessage(ValueError,
                                 "len(count) cannot be larger than 32",
                                 C, a, 33 * [0], symbol)

    def test_canonical_decode_count_range(self):
        a = bitarray()
        for i in range(1, 32):
            count = 32 * [0]
            # negative count
            count[i] = -1
            self.assertRaisesMessage(ValueError,
                "count[%d] not in [0..%d], got -1" % (i, 1 << i),
                canonical_decode, a, count, [])

            maxbits = 1 << i
            count[i] = maxbits
            if i == 31 and PTRSIZE == 4:
                self.assertRaises(OverflowError,
                                  canonical_decode, a, count, [])
                continue
            self.assertRaisesMessage(ValueError,
                "sum(count) = %d, but len(symbol) = 0" % maxbits,
                canonical_decode, a, count, [])

            count[i] = maxbits + 1
            self.assertRaisesMessage(ValueError,
                "count[%d] not in [0..%d], got %d" % (i, maxbits, count[i]),
                canonical_decode, a, count, [])

        iter = canonical_decode(a, 32 * [0], [])
        self.assertEqual(list(iter), [])

    def test_canonical_decode_simple(self):
        C = canonical_decode
        # symbols can be anything, they do not even have to be hashable here
        cnt = [0, 0, 4]
        s = ['A', 42, [1.2-3.7j, 4j], {'B': 6}]
        a = bitarray('00 01 10 11')
        # count can be a list
        self.assertEqual(list(C(a, cnt, s)), s)
        # count can also be a tuple (any sequence object in fact)
        self.assertEqual(list(C(a, (0, 0, 4), s)), s)
        self.assertEqual(list(C(7 * a, cnt, s)), 7 * s)
        # the count list may have extra 0's at the end (but not too many)
        count = [0, 0, 4, 0, 0, 0, 0, 0]
        self.assertEqual(list(C(a, count, s)), s)
        # the element count[0] is unused
        self.assertEqual(list(C(a, [-47, 0, 4], s)), s)
        # in fact it can be anything, as it is entirely ignored
        self.assertEqual(list(C(a, [None, 0, 4], s)), s)

        # the symbol argument can be any sequence object
        s = [65, 66, 67, 98]
        self.assertEqual(list(C(a, cnt, s)), s)
        self.assertEqual(list(C(a, cnt, bytearray(s))), s)
        self.assertEqual(list(C(a, cnt, tuple(s))), s)
        self.assertEqual(list(C(a, cnt, bytes(s))), s)
        # Implementation Note:
        #   The symbol can even be an iterable.  This was done because we
        #   want to use PySequence_Fast in order to convert sequence
        #   objects (like bytes and bytearray) to a list.  This is faster
        #   as all objects are now elements in an array of pointers (as
        #   opposed to having the object's __getitem__ method called on
        #   every iteration).
        self.assertEqual(list(C(a, cnt, iter(s))), s)

    def test_canonical_decode_empty(self):
        a = bitarray()
        # count and symbol are empty, ok because sum([]) == len([])
        self.assertEqual(list(canonical_decode(a, [], [])), [])
        a.append(0)
        self.assertRaisesMessage(ValueError, "reached end of bitarray",
                                 list, canonical_decode(a, [], []))
        a = bitarray(31 * '0')
        self.assertRaisesMessage(ValueError, "ran out of codes",
                                 list, canonical_decode(a, [], []))

    def test_canonical_decode_one_symbol(self):
        symbols = ['A']
        count = [0, 1]
        a = bitarray('000')
        self.assertEqual(list(canonical_decode(a, count, symbols)),
                         3 * symbols)
        a.append(1)
        a.extend(bitarray(10 * '0'))
        iterator = canonical_decode(a, count, symbols)
        self.assertRaisesMessage(ValueError, "reached end of bitarray",
                                 list, iterator)

        a.extend(bitarray(20 * '0'))
        iterator = canonical_decode(a, count, symbols)
        self.assertRaisesMessage(ValueError, "ran out of codes",
                                 list, iterator)

    def test_canonical_decode_large(self):
        with open(__file__, 'rb') as f:
            msg = bytearray(f.read())
        self.assertTrue(len(msg) > 10_000)
        codedict, count, symbol = canonical_huffman(Counter(msg))
        a = bitarray()
        a.encode(codedict, msg)
        self.assertEqual(bytearray(canonical_decode(a, count, symbol)), msg)
        self.check_canonical_code(codedict, count, symbol)

    def test_canonical_decode_symbol_change(self):
        msg = bytearray(b"Hello World!")
        codedict, count, symbol = canonical_huffman(Counter(msg))
        self.check_canonical_code(codedict, count, symbol)
        a = bitarray()
        a.encode(codedict, 10 * msg)

        it = canonical_decode(a, count, symbol)
        def decode_one_msg():
            return bytearray(next(it) for _ in range(len(msg)))

        self.assertEqual(decode_one_msg(), msg)
        symbol[symbol.index(ord("l"))] = ord("k")
        self.assertEqual(decode_one_msg(), bytearray(b"Hekko Workd!"))
        del symbol[:]
        self.assertRaises(IndexError, decode_one_msg)

    def ensure_sorted(self, chc, symbol):
        # ensure codes are sorted
        for i in range(len(symbol) - 1):
            a = chc[symbol[i]]
            b = chc[symbol[i + 1]]
            self.assertTrue(ba2int(a) < ba2int(b))

    def ensure_consecutive(self, chc, count, symbol):
        start = 0
        for nbits, cnt in enumerate(count):
            for i in range(start, start + cnt - 1):
                # ensure two consecutive codes (with same bit length) have
                # consecutive integer values
                a = chc[symbol[i]]
                b = chc[symbol[i + 1]]
                self.assertTrue(len(a) == len(b) == nbits)
                self.assertEqual(ba2int(a) + 1, ba2int(b))
            start += cnt

    def ensure_count(self, chc, count):
        # ensure count list corresponds to length counts from codedict
        maxbits = len(count) - 1
        self.assertEqual(maxbits, max(len(a) for a in chc.values()))
        my_count = (maxbits + 1) * [0]
        for a in chc.values():
            self.assertEqual(a.endian, 'big')
            my_count[len(a)] += 1
        self.assertEqual(my_count, count)

    def ensure_complete(self, count):
        # ensure code is complete and not oversubscribed
        maxbits = len(count) - 1
        x = sum(count[i] << (maxbits - i) for i in range(1, maxbits + 1))
        self.assertEqual(x, 1 << maxbits)

    def ensure_round_trip(self, chc, count, symbol):
        # create a short test message, encode and decode
        msg = choices(symbol, k=10)
        a = bitarray()
        a.encode(chc, msg)
        it = canonical_decode(a, count, symbol)
        # the iterator holds a reference to the bitarray and symbol list
        del a, count, symbol
        self.assertEqual(type(it).__name__, 'canonical_decodeiter')
        self.assertEqual(list(it), msg)

    def check_canonical_code(self, chc, count, symbol):
        self.check_code(chc)
        self.assertTrue(len(chc) == len(symbol) == sum(count))
        self.assertEqual(count[0], 0)  # no codes have length 0
        self.assertEqual(set(chc), set(symbol))
        # the code of the last symbol has all 1 bits
        last = chc[symbol[-1]]
        self.assertTrue(last.all())
        # the code of the first symbol starts with bit 0
        first = chc[symbol[0]]
        self.assertEqual(first[0], 0)

        self.ensure_sorted(chc, symbol)
        self.ensure_consecutive(chc, count, symbol)
        self.ensure_count(chc, count)
        self.ensure_complete(count)
        self.ensure_round_trip(chc, count, symbol)

    def test_small_range(self):
        for n in range(2, 10):
            freq = {i: randrange(10) for i in range(n)}
            self.check_canonical_code(*canonical_huffman(freq))

    def test_simple_counter(self):
        plain = bytearray(b'the quick brown fox jumps over the lazy dog.')
        cnt = Counter(plain)
        self.check_canonical_code(*canonical_huffman(cnt))

    def test_no_comp(self):
        freq = {None: 1, "A": 1}  # None and "A" are not comparable
        self.check_canonical_code(*canonical_huffman(freq))

    def test_balanced(self):
        n = 7
        freq = {}
        for i in range(1 << n):
            freq[i] = 1
        code, count, sym = canonical_huffman(freq)
        self.assertEqual(len(code), 1 << n)
        self.assertTrue(all(len(v) == n for v in code.values()))
        self.check_canonical_code(code, count, sym)

    def test_unbalanced(self):
        n = 32
        freq = {}
        for i in range(n):
            freq[i] = 1 << i
        code, count, sym = canonical_huffman(freq)
        self.assertEqual(len(code), n)
        for i in range(n):
            self.assertEqual(len(code[i]), n - max(1, i))
        self.check_canonical_code(code, count, sym)

# ---------------------------------------------------------------------------

if __name__ == '__main__':
    unittest.main()
