COMPAT: 3.6.1 compat for change in PySlice_GetIndices_Ex

This doesn't actually matter to any tests except for some internal
consistency ones.  Bonus is that it eliminates a warning :<    note
that we aren't actually testing this (yet) on Travis as our 3.6 build
uses conda-forge and 3.6.1 is not there as of yet. Its in defaults
though (and shows up on appveyor build).

Author: Jeff Reback <[email protected]>

Closes #15790 from jreback/py361 and squashes the following commits:

42ddddc [Jeff Reback] change to version < 3
d36902c [Jeff Reback] COMPAT: 3.6.1 compat for change in PySlice_GetIndices_Ex

Author: jreback

pandas/_libs/lib.pyx

@@ -3,6 +3,7 @@ cimport numpy as np
 cimport cython
 import numpy as np
 import sys
+
 cdef bint PY3 = (sys.version_info[0] >= 3)
 
 from numpy cimport *
@@ -26,7 +27,8 @@ from cpython cimport (PyDict_New, PyDict_GetItem, PyDict_SetItem,
                       PyObject_SetAttrString,
                       PyObject_RichCompareBool,
                       PyBytes_GET_SIZE,
-                      PyUnicode_GET_SIZE)
+                      PyUnicode_GET_SIZE,
+                      PyObject)
 
 try:
     from cpython cimport PyString_GET_SIZE
@@ -36,11 +38,10 @@ except ImportError:
 cdef extern from "Python.h":
     Py_ssize_t PY_SSIZE_T_MAX
 
-    ctypedef struct PySliceObject:
-        pass
+cdef extern from "compat_helper.h":
 
-    cdef int PySlice_GetIndicesEx(
-        PySliceObject* s, Py_ssize_t length,
+    cdef int slice_get_indices(
+        PyObject* s, Py_ssize_t length,
         Py_ssize_t *start, Py_ssize_t *stop, Py_ssize_t *step,
         Py_ssize_t *slicelength) except -1
 
@@ -1658,8 +1659,8 @@ cpdef slice_get_indices_ex(slice slc, Py_ssize_t objlen=PY_SSIZE_T_MAX):
     if slc is None:
         raise TypeError("slc should be a slice")
 
-    PySlice_GetIndicesEx(<PySliceObject *>slc, objlen,
-                         &start, &stop, &step, &length)
+    slice_get_indices(<PyObject *>slc, objlen,
+                      &start, &stop, &step, &length)
 
     return start, stop, step, length
 
@@ -1683,8 +1684,8 @@ cpdef Py_ssize_t slice_len(
     if slc is None:
         raise TypeError("slc must be slice")
 
-    PySlice_GetIndicesEx(<PySliceObject *>slc, objlen,
-                         &start, &stop, &step, &length)
+    slice_get_indices(<PyObject *>slc, objlen,
+                      &start, &stop, &step, &length)
 
     return length
 

pandas/_libs/src/compat_helper.h

@@ -0,0 +1,37 @@
+/*
+Copyright (c) 2016, PyData Development Team
+All rights reserved.
+
+Distributed under the terms of the BSD Simplified License.
+
+The full license is in the LICENSE file, distributed with this software.
+*/
+
+#ifndef PANDAS__LIBS_SRC_COMPAT_HELPER_H_
+#define PANDAS__LIBS_SRC_COMPAT_HELPER_H_
+
+#include "Python.h"
+#include "numpy_helper.h"
+
+/*
+PySlice_GetIndicesEx changes signature in PY3
+but 3.6.1 in particular changes the behavior of this function slightly
+https://bugs.python.org/issue27867
+*/
+
+PANDAS_INLINE int slice_get_indices(PyObject *s,
+                                    Py_ssize_t length,
+                                    Py_ssize_t *start,
+                                    Py_ssize_t *stop,
+                                    Py_ssize_t *step,
+                                    Py_ssize_t *slicelength) {
+#if PY_VERSION_HEX >= 0x03000000
+  return PySlice_GetIndicesEx(s, length, start, stop,
+                              step, slicelength);
+#else
+  return PySlice_GetIndicesEx((PySliceObject *)s, length, start,
+                              stop, step, slicelength);
+#endif
+}
+
+#endif  // PANDAS__LIBS_SRC_COMPAT_HELPER_H_

pandas/tests/test_internals.py

@@ -2,11 +2,12 @@
 # pylint: disable=W0102
 
 from datetime import datetime, date
-
+import sys
 import pytest
 import numpy as np
 
 import re
+from distutils.version import LooseVersion
 import itertools
 from pandas import (Index, MultiIndex, DataFrame, DatetimeIndex,
                     Series, Categorical)
@@ -22,6 +23,9 @@
                                  randn, assert_series_equal)
 from pandas.compat import zip, u
 
+# in 3.6.1 a c-api slicing function changed, see src/compat_helper.h
+PY361 = sys.version >= LooseVersion('3.6.1')
+
 
 @pytest.fixture
 def mgr():
@@ -1128,8 +1132,10 @@ def assert_as_slice_equals(arr, slc):
         assert_as_slice_equals([0, 100], slice(0, 200, 100))
 
         assert_as_slice_equals([2, 1], slice(2, 0, -1))
-        assert_as_slice_equals([2, 1, 0], slice(2, None, -1))
-        assert_as_slice_equals([100, 0], slice(100, None, -100))
+
+        if not PY361:
+            assert_as_slice_equals([2, 1, 0], slice(2, None, -1))
+            assert_as_slice_equals([100, 0], slice(100, None, -100))
 
     def test_not_slice_like_arrays(self):
         def assert_not_slice_like(arr):
@@ -1150,8 +1156,9 @@ def test_slice_iter(self):
         assert list(BlockPlacement(slice(0, 0))) == []
         assert list(BlockPlacement(slice(3, 0))) == []
 
-        assert list(BlockPlacement(slice(3, 0, -1))) == [3, 2, 1]
-        assert list(BlockPlacement(slice(3, None, -1))) == [3, 2, 1, 0]
+        if not PY361:
+            assert list(BlockPlacement(slice(3, 0, -1))) == [3, 2, 1]
+            assert list(BlockPlacement(slice(3, None, -1))) == [3, 2, 1, 0]
 
     def test_slice_to_array_conversion(self):
         def assert_as_array_equals(slc, asarray):
@@ -1164,8 +1171,10 @@ def assert_as_array_equals(slc, asarray):
         assert_as_array_equals(slice(3, 0), [])
 
         assert_as_array_equals(slice(3, 0, -1), [3, 2, 1])
-        assert_as_array_equals(slice(3, None, -1), [3, 2, 1, 0])
-        assert_as_array_equals(slice(31, None, -10), [31, 21, 11, 1])
+
+        if not PY361:
+            assert_as_array_equals(slice(3, None, -1), [3, 2, 1, 0])
+            assert_as_array_equals(slice(31, None, -10), [31, 21, 11, 1])
 
     def test_blockplacement_add(self):
         bpl = BlockPlacement(slice(0, 5))
@@ -1180,23 +1189,26 @@ def assert_add_equals(val, inc, result):
         assert_add_equals(slice(0, 0), 0, [])
         assert_add_equals(slice(1, 4), 0, [1, 2, 3])
         assert_add_equals(slice(3, 0, -1), 0, [3, 2, 1])
-        assert_add_equals(slice(2, None, -1), 0, [2, 1, 0])
         assert_add_equals([1, 2, 4], 0, [1, 2, 4])
 
         assert_add_equals(slice(0, 0), 10, [])
         assert_add_equals(slice(1, 4), 10, [11, 12, 13])
         assert_add_equals(slice(3, 0, -1), 10, [13, 12, 11])
-        assert_add_equals(slice(2, None, -1), 10, [12, 11, 10])
         assert_add_equals([1, 2, 4], 10, [11, 12, 14])
 
         assert_add_equals(slice(0, 0), -1, [])
         assert_add_equals(slice(1, 4), -1, [0, 1, 2])
-        assert_add_equals(slice(3, 0, -1), -1, [2, 1, 0])
         assert_add_equals([1, 2, 4], -1, [0, 1, 3])
 
         with pytest.raises(ValueError):
             BlockPlacement(slice(1, 4)).add(-10)
         with pytest.raises(ValueError):
             BlockPlacement([1, 2, 4]).add(-10)
-        with pytest.raises(ValueError):
-            BlockPlacement(slice(2, None, -1)).add(-1)
+
+        if not PY361:
+            assert_add_equals(slice(3, 0, -1), -1, [2, 1, 0])
+            assert_add_equals(slice(2, None, -1), 0, [2, 1, 0])
+            assert_add_equals(slice(2, None, -1), 10, [12, 11, 10])
+
+            with pytest.raises(ValueError):
+                BlockPlacement(slice(2, None, -1)).add(-1)

setup.py

@@ -460,7 +460,8 @@ def pxd(name):
     extra_compile_args=['-Wno-unused-function']
 
 lib_depends = lib_depends + ['pandas/_libs/src/numpy_helper.h',
-                             'pandas/_libs/src/parse_helper.h']
+                             'pandas/_libs/src/parse_helper.h',
+                             'pandas/_libs/src/compat_helper.h']
 
 
 tseries_depends = ['pandas/_libs/src/datetime/np_datetime.h',