Merge pull request #46 from gaoflow/fix-base10-fast-path

Fix base10 dropping leading zeros and corrupting some values on decode

Author: dhondta

src/codext/base/_base.py

@@ -125,8 +125,10 @@ def base_encode(input, charset, errors="strict", exc=BaseEncodeError):
         if i > SIZE_LIMIT:
             raise InputSizeLimitError("Input exceeded size limit")
         return i * charset[0]
-    if n == 10:
-        return str(i) if charset == digits else "".join(charset[int(x)] for x in str(i))
+    # keep this fast-path only for a non-standard 10-character charset ; the digits
+    # charset uses the generic loop below (leading zeros, bignums, no str/int round-trip)
+    if n == 10 and charset != digits:
+        return "".join(charset[int(x)] for x in str(i))
     while i > 0:
         i, c = divmod(i, n)
         r = charset[c] + r
@@ -148,8 +150,8 @@ def base_decode(input, charset, errors="strict", exc=BaseDecodeError):
     i, n, dec = 0, len(charset), lambda n: base_encode(n, [chr(x) for x in range(256)], errors, exc)
     if n == 1:
         return i2s(len(input))
-    if n == 10:
-        return i2s(int(input)) if charset == digits else "".join(str(charset.index(c)) for c in input)
+    if n == 10 and charset != digits:
+        return "".join(str(charset.index(c)) for c in input)
     for k, c in enumerate(input):
         try:
             i = i * n + charset.index(c)

tests/test_base.py

@@ -112,7 +112,30 @@ def test_codec_base8(self):
         self.assertEqual(codecs.decode(B8, "base8-01234567"), STR)
         self.assertRaises(LookupError, codecs.encode, "test", "base8-0123456")
         self.assertRaises(LookupError, codecs.encode, "test", "base8-012345678")
-    
+
+    def test_codec_base10(self):
+        B10 = "2361031878030638688519054699098996"
+        self.assertEqual(codecs.encode(STR, "base10"), B10)
+        self.assertEqual(codecs.encode(b(STR), "base10"), b(B10))
+        self.assertEqual(codecs.decode(B10, "base10"), STR)
+        self.assertEqual(codecs.decode(b(B10), "base10"), b(STR))
+        for alias in ["int", "integer", "dec", "decimal"]:
+            self.assertEqual(codecs.encode(STR, alias), B10)
+            self.assertEqual(codecs.decode(B10, alias), STR)
+        # leading null bytes must be preserved as leading '0' characters
+        self.assertEqual(codecs.encode("\x00abc", "base10"), "06382179")
+        self.assertEqual(codecs.encode("\x00", "base10"), "0")
+        self.assertEqual(codecs.encode("\x00\x00abc", "base10"), "006382179")
+        self.assertEqual(codecs.decode("06382179", "base10"), "\x00abc")
+        self.assertEqual(codecs.decode("00", "base10"), "\x00\x00")
+        self.assertEqual(codecs.encode(b("\x00abc"), "base10"), b("06382179"))
+        self.assertEqual(codecs.decode(b("06382179"), "base10"), b("\x00abc"))
+        # a value whose big integer ends in a 0xe nibble must survive decoding
+        self.assertEqual(codecs.decode(codecs.encode(b"d\xc6\xfe", "base10"), "base10"), b"d\xc6\xfe")
+        # large inputs must not hit the int<->str conversion digit limit
+        for data in [b"\x00\xff\xfe", b"\x00\x00", b"\x2a" * 2048]:
+            self.assertEqual(codecs.decode(codecs.encode(data, "base10"), "base10"), data)
+
     def test_codec_base16(self):
         B16 = "7468697320697320612074657374"
         self.assertEqual(codecs.encode(STR, "base16"), B16)