merge default into py3.5

10 files changed, 54 insertions, 32 deletions

diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py
index 3d5caed8d7..0c8ef3ffcb 100644
--- a/lib_pypy/cffi/cparser.py
+++ b/lib_pypy/cffi/cparser.py
@@ -803,6 +803,16 @@ class Parser(object):
                            "the actual array length in this context"
                            % exprnode.coord.line)
         #
+        if (isinstance(exprnode, pycparser.c_ast.BinaryOp) and
+                exprnode.op == '+'):
+            return (self._parse_constant(exprnode.left) +
+                    self._parse_constant(exprnode.right))
+        #
+        if (isinstance(exprnode, pycparser.c_ast.BinaryOp) and
+                exprnode.op == '-'):
+            return (self._parse_constant(exprnode.left) -
+                    self._parse_constant(exprnode.right))
+        #
         raise FFIError(":%d: unsupported expression: expected a "
                        "simple numeric constant" % exprnode.coord.line)
 
diff --git a/pypy/doc/config/translation.gcrootfinder.txt b/pypy/doc/config/translation.gcrootfinder.txt
index aee7735008..69d271fd3a 100644
--- a/pypy/doc/config/translation.gcrootfinder.txt
+++ b/pypy/doc/config/translation.gcrootfinder.txt
@@ -9,10 +9,8 @@ alternatives:
 - ``--gcrootfinder=asmgcc``: use assembler hackery to find the
   roots directly from the normal stack.  This is a bit faster,
   but platform specific.  It works so far with GCC or MSVC,
-  on i386 and x86-64.  It is tested only on Linux (where it is
-  the default) so other platforms (as well as MSVC) may need
-  various fixes before they can be used.
+  on i386 and x86-64.  It is tested only on Linux 
+  so other platforms (as well as MSVC) may need
+  various fixes before they can be used. Note asmgcc will be deprecated
+  at some future date, and does not work with clang.
 
-You may have to force the use of the shadowstack root finder if
-you are running into troubles or if you insist on translating
-PyPy with other compilers like clang.
diff --git a/pypy/doc/release-v5.7.0.rst b/pypy/doc/release-v5.7.0.rst
index cda5554c80..611ffe6efd 100644
--- a/pypy/doc/release-v5.7.0.rst
+++ b/pypy/doc/release-v5.7.0.rst
@@ -24,6 +24,12 @@ We also backported the ``f""`` formatting from 3.6 (as an expection; otherwise
 CFFI_ has been updated to 1.10, improving an already great package for
 interfacing with C.
 
+We now use shadowstack as our default gcrootfinder_ even on Linux. The
+alternative, asmgcc, will be deprecated at some future point. While about 3%
+slower, shadowstack is much more easily maintained and debuggable. Also,
+the performance of shadowstack has been improved in general: this should
+close the speed gap between Linux and other platforms.
+
 As always, this release fixed many issues and bugs raised by the
 growing community of PyPy users. We strongly recommend updating.
 
@@ -47,6 +53,7 @@ with making RPython's JIT even better.
 .. _`modules`: project-ideas.html#make-more-python-modules-pypy-friendly
 .. _`help`: project-ideas.html
 .. _`these benchmarks show`: https://morepypy.blogspot.com/2017/03/async-http-benchmarks-on-pypy3.html
+.. _gcrootfinder: config/translation.gcrootfinder.html
 
 What is PyPy?
 =============
diff --git a/pypy/module/_multibytecodec/src/cjkcodecs/cjkcodecs.h b/pypy/module/_multibytecodec/src/cjkcodecs/cjkcodecs.h
index ccbf4bbfca..5252f8a1da 100644
--- a/pypy/module/_multibytecodec/src/cjkcodecs/cjkcodecs.h
+++ b/pypy/module/_multibytecodec/src/cjkcodecs/cjkcodecs.h
@@ -201,10 +201,13 @@ struct pair_encodemap {
 
 #define BEGIN_MAPPINGS_LIST /* empty */
 #define MAPPING_ENCONLY(enc)                                            \
+  RPY_EXTERN const struct dbcs_map pypy_cjkmap_##enc;                   \
   const struct dbcs_map pypy_cjkmap_##enc = {#enc, (void*)enc##_encmap, NULL};
 #define MAPPING_DECONLY(enc)                                            \
+  RPY_EXTERN const struct dbcs_map pypy_cjkmap_##enc;                   \
   const struct dbcs_map pypy_cjkmap_##enc = {#enc, NULL, (void*)enc##_decmap};
 #define MAPPING_ENCDEC(enc)                                             \
+  RPY_EXTERN const struct dbcs_map pypy_cjkmap_##enc;                   \
   const struct dbcs_map pypy_cjkmap_##enc = {#enc, (void*)enc##_encmap, \
                                              (void*)enc##_decmap};
 #define END_MAPPINGS_LIST /* empty */
@@ -294,7 +297,7 @@ find_pairencmap(ucs2_t body, ucs2_t modifier,
 
 #ifdef USING_IMPORTED_MAPS
 #define USING_IMPORTED_MAP(charset) \
-  extern const struct dbcs_map pypy_cjkmap_##charset;
+  RPY_EXTERN const struct dbcs_map pypy_cjkmap_##charset;
 
 #define IMPORT_MAP(locale, charset, encmap, decmap)                     \
   importmap(&pypy_cjkmap_##charset, encmap, decmap)
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py
index 7198d923af..caceb3206d 100644
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py
@@ -387,13 +387,14 @@ def test_const_pointer_to_pointer():
 def test_enum():
     ffi = FFI()
     ffi.cdef("""
-        enum Enum { POS = +1, TWO = 2, NIL = 0, NEG = -1};
+        enum Enum { POS = +1, TWO = 2, NIL = 0, NEG = -1, OP = (POS+TWO)-1};
         """)
     C = ffi.dlopen(None)
     assert C.POS == 1
     assert C.TWO == 2
     assert C.NIL == 0
     assert C.NEG == -1
+    assert C.OP == 2
 
 def test_stdcall():
     ffi = FFI()
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
index 172e4c4b99..328ca3b4e0 100644
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -237,6 +237,8 @@ else:
                 'sys/resource.h',
                 'grp.h', 'dirent.h', 'sys/stat.h', 'fcntl.h',
                 'signal.h', 'sys/utsname.h', _ptyh]
+    if sys.platform.startswith('linux'):
+        includes.append('sys/sysmacros.h')
     if sys.platform.startswith('freebsd'):
         includes.append('sys/ttycom.h')
     libraries = ['util']
diff --git a/rpython/rlib/rsiphash.py b/rpython/rlib/rsiphash.py
index 921341509c..485f03e1cc 100644
--- a/rpython/rlib/rsiphash.py
+++ b/rpython/rlib/rsiphash.py
@@ -24,14 +24,11 @@ from rpython.rtyper.annlowlevel import llhelper
 if sys.byteorder == 'little':
     def _le64toh(x):
         return x
+    def _le32toh(x):
+        return x
 else:
     _le64toh = rarithmetic.byteswap
-
-
-class Seed:
-    k0l = k1l = r_uint64(0)
-seed = Seed()
-
+    _le32toh = rarithmetic.byteswap
 
 def _decode64(s):
     return (r_uint64(ord(s[0])) |
@@ -43,6 +40,11 @@ def _decode64(s):
             r_uint64(ord(s[6])) << 48 |
             r_uint64(ord(s[7])) << 56)
 
+class Seed:
+    k0l = k1l = r_uint64(0)
+seed = Seed()
+
+
 def select_random_seed(s):
     """'s' is a string of length 16"""
     seed.k0l = _decode64(s)
@@ -177,17 +179,13 @@ def choosen_seed(new_k0, new_k1, test_misaligned_path=False):
     """For tests."""
     global misaligned_is_fine
     old = seed.k0l, seed.k1l, misaligned_is_fine
-    seed.k0l = _le64toh(r_uint64(new_k0))
-    seed.k1l = _le64toh(r_uint64(new_k1))
+    seed.k0l = r_uint64(new_k0)
+    seed.k1l = r_uint64(new_k1)
     if test_misaligned_path:
         misaligned_is_fine = False
     yield
     seed.k0l, seed.k1l, misaligned_is_fine = old
 
-def get_current_seed():
-    return _le64toh(seed.k0l), _le64toh(seed.k1l)
-
-
 magic0 = r_uint64(0x736f6d6570736575)
 magic1 = r_uint64(0x646f72616e646f6d)
 magic2 = r_uint64(0x6c7967656e657261)
@@ -271,7 +269,8 @@ def _siphash24(addr_in, size):
         size = 4
     if size == 4:
         if direct:
-            t |= r_uint64(llop.raw_load(rffi.UINT, addr_in, index))
+            v = _le32toh(r_uint32(llop.raw_load(rffi.UINT, addr_in, index)))
+            t |= r_uint64(v)
             size = 0
         else:
             t |= r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 3)) << 24
@@ -287,7 +286,7 @@ def _siphash24(addr_in, size):
         size = 0
     assert size == 0
 
-    b |= _le64toh(t)
+    b |= t
 
     v3 ^= b
     v0, v1, v2, v3 = _double_round(v0, v1, v2, v3)
diff --git a/rpython/rlib/rvmprof/src/vmprof_getpc.h b/rpython/rlib/rvmprof/src/vmprof_getpc.h
index 3e970e5289..641a11d416 100644
--- a/rpython/rlib/rvmprof/src/vmprof_getpc.h
+++ b/rpython/rlib/rvmprof/src/vmprof_getpc.h
@@ -131,7 +131,7 @@ struct CallUnrollInfo {
 // typedef int ucontext_t;
 // #endif
 
-intptr_t GetPC(ucontext_t *signal_ucontext) {
+static intptr_t GetPC(ucontext_t *signal_ucontext) {
   // RAW_LOG(ERROR, "GetPC is not yet implemented on Windows\n");
   fprintf(stderr, "GetPC is not yet implemented on Windows\n");
   return NULL;
@@ -142,7 +142,7 @@ intptr_t GetPC(ucontext_t *signal_ucontext) {
 // the right value for your system, and add it to the list in
 // vmrpof_config.h
 #else
-intptr_t GetPC(ucontext_t *signal_ucontext) {
+static intptr_t GetPC(ucontext_t *signal_ucontext) {
   return signal_ucontext->PC_FROM_UCONTEXT;   // defined in config.h
 }
 
diff --git a/rpython/rlib/rvmprof/src/vmprof_main.h b/rpython/rlib/rvmprof/src/vmprof_main.h
index c26918c582..4434f1c877 100644
--- a/rpython/rlib/rvmprof/src/vmprof_main.h
+++ b/rpython/rlib/rvmprof/src/vmprof_main.h
@@ -104,8 +104,8 @@ static intptr_t get_current_thread_id(void)
 
 #include <setjmp.h>
 
-volatile int spinlock;
-jmp_buf restore_point;
+static volatile int spinlock;
+static jmp_buf restore_point;
 
 static void segfault_handler(int arg)
 {
diff --git a/rpython/rlib/test/test_rsiphash.py b/rpython/rlib/test/test_rsiphash.py
index a93e2a9608..ffd09fe941 100644
--- a/rpython/rlib/test/test_rsiphash.py
+++ b/rpython/rlib/test/test_rsiphash.py
@@ -52,12 +52,12 @@ def test_fix_seed():
         os.environ['PYTHONHASHSEED'] = '0'
         initialize_from_env()
         assert siphash24("foo") == 15988776847138518036
-        # value checked with CPython 3.5
+        # value checked with CPython 3.5 (turned positive by adding 2**64)
 
         os.environ['PYTHONHASHSEED'] = '4000000000'
         initialize_from_env()
         assert siphash24("foo") == 13829150778707464258
-        # value checked with CPython 3.5
+        # value checked with CPython 3.5 (turned positive by adding 2**64)
 
         for env in ['', 'random']:
             os.environ['PYTHONHASHSEED'] = env
@@ -118,8 +118,9 @@ def test_translated():
             123, 123, intmask(15988776847138518036),
             456, 456, intmask(15988776847138518036),
             789, 789]
-        assert s1[8] in [intmask(17593683438421985039),    # ucs2 mode
-                         intmask(94801584261658677)]       # ucs4 mode
+        assert s1[8] in [intmask(17593683438421985039),    # ucs2 mode little endian
+                         intmask(94801584261658677),       # ucs4 mode little endian
+                         intmask(3849431280840015342),]    # ucs4 mode big endian
 
         os.environ['PYTHONHASHSEED'] = '3987654321'
         s1 = getall()
@@ -127,8 +128,9 @@ def test_translated():
             123, 123, intmask(5890804383681474441),
             456, 456, intmask(5890804383681474441),
             789, 789]
-        assert s1[8] in [intmask(4192582507672183374),     # ucs2 mode
-                         intmask(7179255293164649778)]     # ucs4 mode
+        assert s1[8] in [intmask(4192582507672183374),     # ucs2 mode little endian
+                         intmask(7179255293164649778),     # ucs4 mode little endian
+                         intmask(-3945781295304514711),]   # ucs4 mode big endian
 
         for env in ['', 'random']:
             os.environ['PYTHONHASHSEED'] = env