BLD: pandas imports. lot of stuff still broken though

Author: wesm

Makefile

@@ -1,7 +1,7 @@
 clean:
 	-rm -rf build dist
 
-tseries: pandas/src/tseries.pyx
+tseries: pandas/lib.pyx pandas/tslib.pyx pandas/hashtable.pyx
 	python setup.py build_ext --inplace
 
 sparse: pandas/src/sparse.pyx

pandas/__init__.py

@@ -7,7 +7,9 @@
 import numpy as np
 
 try:
-    import pandas.lib as lib
+    import hashtable
+    import tslib
+    import lib
 except Exception:  # pragma: no cover
     import sys
     e = sys.exc_info()[1] # Py25 and Py3 current exception syntax conflict

pandas/src/stats.pyx renamed to pandas/algos.pyx

@@ -1,5 +1,6 @@
 from numpy cimport *
 cimport numpy as np
+import numpy as np
 
 cimport cython
 
@@ -1707,6 +1708,83 @@ def roll_generic(ndarray[float64_t, cast=True] input, int win,
 #----------------------------------------------------------------------
 # group operations
 
+@cython.boundscheck(False)
+def groupby_indices(ndarray values):
+    cdef:
+        Py_ssize_t i, n = len(values)
+        ndarray[int64_t] labels, counts, arr, seen
+        int64_t loc
+        dict ids = {}
+        object val
+        int64_t k
+
+    ids, labels, counts = group_labels(values)
+    seen = np.zeros_like(counts)
+
+    # try not to get in trouble here...
+    cdef int64_t **vecs = <int64_t **> malloc(len(ids) * sizeof(int64_t*))
+    result = {}
+    for i from 0 <= i < len(counts):
+        arr = np.empty(counts[i], dtype=np.int64)
+        result[ids[i]] = arr
+        vecs[i] = <int64_t *> arr.data
+
+    for i from 0 <= i < n:
+        k = labels[i]
+
+        # was NaN
+        if k == -1:
+            continue
+
+        loc = seen[k]
+        vecs[k][loc] = i
+        seen[k] = loc + 1
+
+    free(vecs)
+
+    return result
+
+@cython.wraparound(False)
+@cython.boundscheck(False)
+def group_labels(ndarray[object] values):
+    '''
+    Compute label vector from input values and associated useful data
+
+    Returns
+    -------
+    '''
+    cdef:
+        Py_ssize_t i, n = len(values)
+        ndarray[int64_t] labels = np.empty(n, dtype=np.int64)
+        ndarray[int64_t] counts = np.empty(n, dtype=np.int64)
+        dict ids = {}, reverse = {}
+        int64_t idx
+        object val
+        int64_t count = 0
+
+    for i from 0 <= i < n:
+        val = values[i]
+
+        # is NaN
+        if val != val:
+            labels[i] = -1
+            continue
+
+        # for large number of groups, not doing try: except: makes a big
+        # difference
+        if val in ids:
+            idx = ids[val]
+            labels[i] = idx
+            counts[idx] = counts[idx] + 1
+        else:
+            ids[val] = count
+            reverse[count] = val
+            labels[i] = count
+            counts[count] = 1
+            count += 1
+
+    return reverse, labels, counts[:count].copy()
+
 
 @cython.boundscheck(False)
 @cython.wraparound(False)
@@ -2943,3 +3021,4 @@ def group_var_bin(ndarray[float64_t, ndim=2] out,
                              (ct * ct - ct))
 
 include "join.pyx"
+include "generated.pyx"

pandas/core/algorithms.py

@@ -6,8 +6,8 @@
 import numpy as np
 
 import pandas.core.common as com
-import pandas.lib as lib
-import pandas._algos as _algos
+import pandas.algos as algos
+import pandas.hashtable as htable
 
 
 def match(to_match, values, na_sentinel=-1):
@@ -70,11 +70,11 @@ def _hashtable_algo(f, dtype):
     f(HashTable, type_caster) -> result
     """
     if com.is_float_dtype(dtype):
-        return f(lib.Float64HashTable, com._ensure_float64)
+        return f(htable.Float64HashTable, com._ensure_float64)
     elif com.is_integer_dtype(dtype):
-        return f(lib.Int64HashTable, com._ensure_int64)
+        return f(htable.Int64HashTable, com._ensure_int64)
     else:
-        return f(lib.PyObjectHashTable, com._ensure_object)
+        return f(htable.PyObjectHashTable, com._ensure_object)
 
 
 def _count_generic(values, table_type, type_caster):
@@ -167,7 +167,7 @@ def value_counts(values, sort=True, ascending=False):
 
     if com.is_integer_dtype(values.dtype):
         values = com._ensure_int64(values)
-        keys, counts = lib.value_count_int64(values)
+        keys, counts = htable.value_count_int64(values)
         result = Series(counts, index=keys)
     else:
         counter = defaultdict(lambda: 0)
@@ -271,7 +271,7 @@ def _get_score(at):
         return _get_score(q)
     else:
         q = np.asarray(q, np.float64)
-        return _algos.arrmap_float64(q, _get_score)
+        return algos.arrmap_float64(q, _get_score)
 
 
 def _interpolate(a, b, fraction):
@@ -313,19 +313,19 @@ def group_position(*args):
 
 
 _rank1d_functions = {
-    'float64': lib.rank_1d_float64,
-    'int64': lib.rank_1d_int64,
-    'generic': lib.rank_1d_generic
+    'float64': algos.rank_1d_float64,
+    'int64': algos.rank_1d_int64,
+    'generic': algos.rank_1d_generic
 }
 
 _rank2d_functions = {
-    'float64': lib.rank_2d_float64,
-    'int64': lib.rank_2d_int64,
-    'generic': lib.rank_2d_generic
+    'float64': algos.rank_2d_float64,
+    'int64': algos.rank_2d_int64,
+    'generic': algos.rank_2d_generic
 }
 
 _hashtables = {
-    'float64': (lib.Float64HashTable, lib.Float64Vector),
-    'int64': (lib.Int64HashTable, lib.Int64Vector),
-    'generic': (lib.PyObjectHashTable, lib.ObjectVector)
+    'float64': (htable.Float64HashTable, htable.Float64Vector),
+    'int64': (htable.Int64HashTable, htable.Int64Vector),
+    'generic': (htable.PyObjectHashTable, htable.ObjectVector)
 }

pandas/core/common.py

@@ -11,8 +11,10 @@
 from numpy.lib.format import read_array, write_array
 import numpy as np
 
-import pandas._algos as _algos
+import pandas.algos as algos
 import pandas.lib as lib
+import pandas.tslib as tslib
+
 from pandas.util import py3compat
 import codecs
 import csv
@@ -84,7 +86,7 @@ def _isnull_ndarraylike(obj):
             result = Series(result, index=obj.index, copy=False)
     elif values.dtype == np.dtype('M8[ns]'):
         # this is the NaT pattern
-        result = values.view('i8') == lib.iNaT
+        result = values.view('i8') == tslib.iNaT
     elif issubclass(values.dtype.type, np.timedelta64):
         result = -np.isfinite(values.view('i8'))
     else:
@@ -168,43 +170,43 @@ def wrapper(arr, indexer, out, fill_value=np.nan):
 
 
 _take1d_dict = {
-    'float64': _algos.take_1d_float64,
-    'int32': _algos.take_1d_int32,
-    'int64': _algos.take_1d_int64,
-    'object': _algos.take_1d_object,
-    'bool': _view_wrapper(_algos.take_1d_bool, np.uint8),
-    'datetime64[ns]': _view_wrapper(_algos.take_1d_int64, np.int64,
-                                    na_override=lib.iNaT),
+    'float64': algos.take_1d_float64,
+    'int32': algos.take_1d_int32,
+    'int64': algos.take_1d_int64,
+    'object': algos.take_1d_object,
+    'bool': _view_wrapper(algos.take_1d_bool, np.uint8),
+    'datetime64[ns]': _view_wrapper(algos.take_1d_int64, np.int64,
+                                    na_override=tslib.iNaT),
 }
 
 _take2d_axis0_dict = {
-    'float64': _algos.take_2d_axis0_float64,
-    'int32': _algos.take_2d_axis0_int32,
-    'int64': _algos.take_2d_axis0_int64,
-    'object': _algos.take_2d_axis0_object,
-    'bool': _view_wrapper(_algos.take_2d_axis0_bool, np.uint8),
-    'datetime64[ns]': _view_wrapper(_algos.take_2d_axis0_int64, np.int64,
-                                    na_override=lib.iNaT),
+    'float64': algos.take_2d_axis0_float64,
+    'int32': algos.take_2d_axis0_int32,
+    'int64': algos.take_2d_axis0_int64,
+    'object': algos.take_2d_axis0_object,
+    'bool': _view_wrapper(algos.take_2d_axis0_bool, np.uint8),
+    'datetime64[ns]': _view_wrapper(algos.take_2d_axis0_int64, np.int64,
+                                    na_override=tslib.iNaT),
 }
 
 _take2d_axis1_dict = {
-    'float64': _algos.take_2d_axis1_float64,
-    'int32': _algos.take_2d_axis1_int32,
-    'int64': _algos.take_2d_axis1_int64,
-    'object': _algos.take_2d_axis1_object,
-    'bool': _view_wrapper(_algos.take_2d_axis1_bool, np.uint8),
-    'datetime64[ns]': _view_wrapper(_algos.take_2d_axis1_int64, np.int64,
-                                     na_override=lib.iNaT),
+    'float64': algos.take_2d_axis1_float64,
+    'int32': algos.take_2d_axis1_int32,
+    'int64': algos.take_2d_axis1_int64,
+    'object': algos.take_2d_axis1_object,
+    'bool': _view_wrapper(algos.take_2d_axis1_bool, np.uint8),
+    'datetime64[ns]': _view_wrapper(algos.take_2d_axis1_int64, np.int64,
+                                     na_override=tslib.iNaT),
 }
 
 _take2d_multi_dict = {
-    'float64': _algos.take_2d_multi_float64,
-    'int32': _algos.take_2d_multi_int32,
-    'int64': _algos.take_2d_multi_int64,
-    'object': _algos.take_2d_multi_object,
-    'bool': _view_wrapper(_algos.take_2d_multi_bool, np.uint8),
-    'datetime64[ns]': _view_wrapper(_algos.take_2d_multi_int64, np.int64,
-                                    na_override=lib.iNaT),
+    'float64': algos.take_2d_multi_float64,
+    'int32': algos.take_2d_multi_int32,
+    'int64': algos.take_2d_multi_int64,
+    'object': algos.take_2d_multi_object,
+    'bool': _view_wrapper(algos.take_2d_multi_bool, np.uint8),
+    'datetime64[ns]': _view_wrapper(algos.take_2d_multi_int64, np.int64,
+                                    na_override=tslib.iNaT),
 }
 
 
@@ -369,9 +371,9 @@ def mask_out_axis(arr, mask, axis, fill_value=np.nan):
     arr[tuple(indexer)] = fill_value
 
 _diff_special = {
-    'float64': lib.diff_2d_float64,
-    'int64': lib.diff_2d_int64,
-    'int32': lib.diff_2d_int32
+    'float64': algos.diff_2d_float64,
+    'int64': algos.diff_2d_int64,
+    'int32': algos.diff_2d_int32
 }
 
 def diff(arr, n, axis=0):
@@ -452,21 +454,21 @@ def wrapper(arr, mask, limit=None):
         f(view, mask, limit=limit)
     return wrapper
 
-_pad_1d_datetime = _interp_wrapper(_algos.pad_inplace_int64, np.int64)
-_pad_2d_datetime = _interp_wrapper(_algos.pad_2d_inplace_int64, np.int64)
-_backfill_1d_datetime = _interp_wrapper(_algos.backfill_inplace_int64,
+_pad_1d_datetime = _interp_wrapper(algos.pad_inplace_int64, np.int64)
+_pad_2d_datetime = _interp_wrapper(algos.pad_2d_inplace_int64, np.int64)
+_backfill_1d_datetime = _interp_wrapper(algos.backfill_inplace_int64,
                                         np.int64)
-_backfill_2d_datetime = _interp_wrapper(_algos.backfill_2d_inplace_int64,
+_backfill_2d_datetime = _interp_wrapper(algos.backfill_2d_inplace_int64,
                                         np.int64)
 
 
 def pad_1d(values, limit=None, mask=None):
     if is_float_dtype(values):
-        _method = _algos.pad_inplace_float64
+        _method = algos.pad_inplace_float64
     elif is_datetime64_dtype(values):
         _method = _pad_1d_datetime
     elif values.dtype == np.object_:
-        _method = _algos.pad_inplace_object
+        _method = algos.pad_inplace_object
     else:  # pragma: no cover
         raise ValueError('Invalid dtype for padding')
 
@@ -478,11 +480,11 @@ def pad_1d(values, limit=None, mask=None):
 
 def backfill_1d(values, limit=None, mask=None):
     if is_float_dtype(values):
-        _method = _algos.backfill_inplace_float64
+        _method = algos.backfill_inplace_float64
     elif is_datetime64_dtype(values):
         _method = _backfill_1d_datetime
     elif values.dtype == np.object_:
-        _method = _algos.backfill_inplace_object
+        _method = algos.backfill_inplace_object
     else:  # pragma: no cover
         raise ValueError('Invalid dtype for padding')
 
@@ -495,11 +497,11 @@ def backfill_1d(values, limit=None, mask=None):
 
 def pad_2d(values, limit=None, mask=None):
     if is_float_dtype(values):
-        _method = _algos.pad_2d_inplace_float64
+        _method = algos.pad_2d_inplace_float64
     elif is_datetime64_dtype(values):
         _method = _pad_2d_datetime
     elif values.dtype == np.object_:
-        _method = _algos.pad_2d_inplace_object
+        _method = algos.pad_2d_inplace_object
     else:  # pragma: no cover
         raise ValueError('Invalid dtype for padding')
 
@@ -516,11 +518,11 @@ def pad_2d(values, limit=None, mask=None):
 
 def backfill_2d(values, limit=None, mask=None):
     if is_float_dtype(values):
-        _method = _algos.backfill_2d_inplace_float64
+        _method = algos.backfill_2d_inplace_float64
     elif is_datetime64_dtype(values):
         _method = _backfill_2d_datetime
     elif values.dtype == np.object_:
-        _method = _algos.backfill_2d_inplace_object
+        _method = algos.backfill_2d_inplace_object
     else:  # pragma: no cover
         raise ValueError('Invalid dtype for padding')
 
@@ -903,11 +905,11 @@ def _is_sequence(x):
     except Exception:
         return False
 
-_ensure_float64 = _algos.ensure_float64
-_ensure_int64 = _algos.ensure_int64
-_ensure_int32 = _algos.ensure_int32
-_ensure_platform_int = _algos.ensure_platform_int
-_ensure_object = _algos.ensure_object
+_ensure_float64 = algos.ensure_float64
+_ensure_int64 = algos.ensure_int64
+_ensure_int32 = algos.ensure_int32
+_ensure_platform_int = algos.ensure_platform_int
+_ensure_object = algos.ensure_object
 
 
 def _astype_nansafe(arr, dtype):
@@ -916,7 +918,7 @@ def _astype_nansafe(arr, dtype):
 
     if issubclass(arr.dtype.type, np.datetime64):
         if dtype == object:
-            return lib.ints_to_pydatetime(arr.view(np.int64))
+            return tslib.ints_to_pydatetime(arr.view(np.int64))
     elif (np.issubdtype(arr.dtype, np.floating) and
         np.issubdtype(dtype, np.integer)):