Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

RFC: clean up map and collect with iterator traits #15146

Merged
merged 3 commits into from
Mar 29, 2016
Merged

RFC: clean up map and collect with iterator traits #15146

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
a2ce230
make `map` and `collect` more general and uniform by adding iterator …
JeffBezanson Feb 17, 2016
d60e002
zips, products and takes with infinite components
mschauer Mar 8, 2016
bd02ba4
length() of Take and Drop iterators
mschauer Mar 10, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
152 changes: 5 additions & 147 deletions base/abstractarray.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1086,34 +1086,6 @@ foreach(f) = (f(); nothing)
foreach(f, itr) = (for x in itr; f(x); end; nothing)
foreach(f, itrs...) = (for z in zip(itrs...); f(z...); end; nothing)

# generic map on any iterator
function map(f, iters...)
result = []
len = length(iters)
states = [start(iters[idx]) for idx in 1:len]
nxtvals = cell(len)
cont = true
for idx in 1:len
if done(iters[idx], states[idx])
cont = false
break
end
end
while cont
for idx in 1:len
nxtvals[idx],states[idx] = next(iters[idx], states[idx])
end
push!(result, f(nxtvals...))
for idx in 1:len
if done(iters[idx], states[idx])
cont = false
break
end
end
end
result
end

## map over arrays ##

## transform any set of dimensions
Expand Down Expand Up @@ -1174,39 +1146,6 @@ function mapslices(f, A::AbstractArray, dims::AbstractVector)
return R
end


# using promote_type
function promote_to!{T,F}(f::F, offs, dest::AbstractArray{T}, A::AbstractArray)
# map to dest array, checking the type of each result. if a result does not
# match, do a type promotion and re-dispatch.
for i = offs:length(A) #Fixme iter
@inbounds Ai = A[i]
el = f(Ai)
S = typeof(el)
if S === T || S <: T
@inbounds dest[i] = el::T
else
R = promote_type(T, S)
if R !== T
new = similar(dest, R)
copy!(new,1, dest,1, i-1)
new[i] = el
return promote_to!(f, i+1, new, A)
end
@inbounds dest[i] = el
end
end
return dest
end

function map_promote(f, A::AbstractArray)
if isempty(A); return similar(A, Bottom); end
first = f(A[1])
dest = similar(A, typeof(first))
dest[1] = first
return promote_to!(f, 2, dest, A)
end

# These are needed because map(eltype, As) is not inferrable
promote_eltype_op(::Any) = (@_pure_meta; Bottom)
promote_eltype_op{T}(op, ::AbstractArray{T}) = (@_pure_meta; promote_op(op, T))
Expand All @@ -1225,39 +1164,11 @@ function map!{F}(f::F, dest::AbstractArray, A::AbstractArray)
return dest
end

function map_to!{T,F}(f::F, offs, st, dest::AbstractArray{T}, A)
# map to dest array, checking the type of each result. if a result does not
# match, widen the result type and re-dispatch.
i = offs
while !done(A, st)
@inbounds Ai, st = next(A, st)
el = f(Ai)
S = typeof(el)
if S === T || S <: T
@inbounds dest[i] = el::T
i += 1
else
R = typejoin(T, S)
new = similar(dest, R)
copy!(new,1, dest,1, i-1)
@inbounds new[i] = el
return map_to!(f, i+1, st, new, A)
end
end
return dest
end
# map on collections
map(f, A::Union{AbstractArray,AbstractSet,Associative}) = collect_similar(A, Generator(f,A))

function map(f, A::AbstractArray)
if isempty(A)
return isa(f,Type) ? similar(A,f) : similar(A)
end
st = start(A)
A1, st = next(A, st)
first = f(A1)
dest = similar(A, typeof(first))
dest[1] = first
return map_to!(f, 2, st, dest, A)
end
# default to returning an Array for `map` on general iterators
map(f, A) = collect(Generator(f,A))

## 2 argument
function map!{F}(f::F, dest::AbstractArray, A::AbstractArray, B::AbstractArray)
Expand All @@ -1267,34 +1178,6 @@ function map!{F}(f::F, dest::AbstractArray, A::AbstractArray, B::AbstractArray)
return dest
end

function map_to!{T,F}(f::F, offs, dest::AbstractArray{T}, A::AbstractArray, B::AbstractArray)
for i = offs:length(A) #Fixme iter
@inbounds Ai, Bi = A[i], B[i]
el = f(Ai, Bi)
S = typeof(el)
if (S !== T) && !(S <: T)
R = typejoin(T, S)
new = similar(dest, R)
copy!(new,1, dest,1, i-1)
@inbounds new[i] = el
return map_to!(f, i+1, new, A, B)
end
@inbounds dest[i] = el::T
end
return dest
end

function map(f, A::AbstractArray, B::AbstractArray)
shp = promote_shape(size(A),size(B))
if prod(shp) == 0
return similar(A, promote_type(eltype(A),eltype(B)), shp)
end
first = f(A[1], B[1])
dest = similar(A, typeof(first), shp)
dest[1] = first
return map_to!(f, 2, dest, A, B)
end

## N argument

ith_all(i, ::Tuple{}) = ()
Expand All @@ -1310,32 +1193,7 @@ end

map!{F}(f::F, dest::AbstractArray, As::AbstractArray...) = map_n!(f, dest, As)

function map_to_n!{T,F}(f::F, offs, dest::AbstractArray{T}, As)
for i = offs:length(As[1])
el = f(ith_all(i, As)...)
S = typeof(el)
if (S !== T) && !(S <: T)
R = typejoin(T, S)
new = similar(dest, R)
copy!(new,1, dest,1, i-1)
@inbounds new[i] = el
return map_to_n!(f, i+1, new, As)
end
@inbounds dest[i] = el::T
end
return dest
end

function map(f, As::AbstractArray...)
shape = mapreduce(size, promote_shape, As)
if prod(shape) == 0
return similar(As[1], promote_eltype(As...), shape)
end
first = f(map(a->a[1], As)...)
dest = similar(As[1], typeof(first), shape)
dest[1] = first
return map_to_n!(f, 2, dest, As)
end
map(f, iters...) = collect(Generator(f, iters...))

# multi-item push!, unshift! (built on top of type-specific 1-item version)
# (note: must not cause a dispatch loop when 1-item case is not defined)
Expand Down
100 changes: 82 additions & 18 deletions base/array.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -199,35 +199,99 @@ convert{T,n,S}(::Type{Array{T,n}}, x::AbstractArray{S,n}) = copy!(Array(T, size(

promote_rule{T,n,S}(::Type{Array{T,n}}, ::Type{Array{S,n}}) = Array{promote_type(T,S),n}

## copying iterators to containers

# make a collection similar to `c` and appropriate for collecting `itr`
_similar_for(c::AbstractArray, T, itr, ::SizeUnknown) = similar(c, T, 0)
_similar_for(c::AbstractArray, T, itr, ::HasLength) = similar(c, T, Int(length(itr)::Integer))
_similar_for(c::AbstractArray, T, itr, ::HasShape) = similar(c, T, convert(Dims,size(itr)))
_similar_for(c, T, itr, isz) = similar(c, T)

"""
collect(element_type, collection)

Return an array of type `Array{element_type,1}` of all items in a collection.
"""
function collect{T}(::Type{T}, itr)
if applicable(length, itr)
# when length() isn't defined this branch might pollute the
# type of the other.
a = Array(T,length(itr)::Integer)
i = 0
for x in itr
a[i+=1] = x
end
else
a = Array(T,0)
for x in itr
push!(a,x)
end
end
return a
end
collect{T}(::Type{T}, itr) = collect(Generator(T, itr))

"""
collect(collection)

Return an array of all items in a collection. For associative collections, returns Pair{KeyType, ValType}.
"""
collect(itr) = collect(eltype(itr), itr)
collect(itr) = _collect(1:1 #= Array =#, itr, iteratoreltype(itr), iteratorsize(itr))

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

collect(repeated(1)) now gives a method-error instead of hanging? that's nice

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, I suppose so. Would you like to take a stab at adding more support for IsInfinite? I think mostly we need definitions for combining it correctly with other iterators.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, I plan to do this, I am reading now to see how it works.

collect_similar(cont, itr) = _collect(cont, itr, iteratoreltype(itr), iteratorsize(itr))

_collect(cont, itr, ::HasEltype, isz::Union{HasLength,HasShape}) =
copy!(_similar_for(cont, eltype(itr), itr, isz), itr)

function _collect(cont, itr, ::HasEltype, isz::SizeUnknown)
a = _similar_for(cont, eltype(itr), itr, isz)
for x in itr
push!(a,x)
end
return a
end

_collect(c, itr, ::EltypeUnknown, isz::SizeUnknown) = grow_to!(_similar_for(c, Union{}, itr, isz), itr)

function _collect(c, itr, ::EltypeUnknown, isz::Union{HasLength,HasShape})
st = start(itr)
if done(itr,st)
return _similar_for(c, Union{}, itr, isz)
end
v1, st = next(itr, st)
collect_to_with_first!(_similar_for(c, typeof(v1), itr, isz), v1, itr, st)
end

function collect_to_with_first!(dest::AbstractArray, v1, itr, st)
dest[1] = v1
return collect_to!(dest, itr, 2, st)
end

function collect_to_with_first!(dest, v1, itr, st)
push!(dest, v1)
return grow_to!(dest, itr, st)
end

function collect_to!{T}(dest::AbstractArray{T}, itr, offs, st)
# collect to dest array, checking the type of each result. if a result does not
# match, widen the result type and re-dispatch.
i = offs
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

A big 👍 to moving this from map to collect. Really nice.

while !done(itr, st)
el, st = next(itr, st)
S = typeof(el)
if S === T || S <: T
@inbounds dest[i] = el::T
i += 1
else
R = typejoin(T, S)
new = similar(dest, R)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

the syntax highlighting is suggesting using a different name than new for this

copy!(new,1, dest,1, i-1)
@inbounds new[i] = el
return collect_to!(new, itr, i+1, st)
end
end
return dest
end

function grow_to!(dest, itr, st = start(itr))
T = eltype(dest)
while !done(itr, st)
el, st = next(itr, st)
S = typeof(el)
if S === T || S <: T
push!(dest, el::T)
else
new = similar(dest, typejoin(T, S))
copy!(new, dest)
push!(new, el)
return grow_to!(new, itr, st)
end
end
return dest
end

## Iteration ##
start(A::Array) = 1
Expand Down
2 changes: 2 additions & 0 deletions base/channels.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,3 +95,5 @@ function done(c::Channel, state::Ref)
end
end
next{T}(c::Channel{T}, state) = (v=get(state[]); state[]=nothing; (v, state))

iteratorsize{C<:Channel}(::Type{C}) = SizeUnknown()
45 changes: 29 additions & 16 deletions base/dict.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,6 @@

# generic operations on associative collections

abstract Associative{K,V}

const secret_table_token = :__c782dbf1cf4d6a2e5e3865d7e95634f2e09b5902__

haskey(d::Associative, k) = in(k,keys(d))
Expand Down Expand Up @@ -225,6 +223,16 @@ function merge!(d::Associative, others::Associative...)
end
return d
end

# very similar to `merge!`, but accepts any iterable and extends code
# that would otherwise only use `copy!` with arrays.
function copy!(dest::Union{Associative,AbstractSet}, src)
for x in src
push!(dest, x)
end
return dest
end

keytype{K,V}(::Type{Associative{K,V}}) = K
keytype(a::Associative) = keytype(typeof(a))
keytype{A<:Associative}(::Type{A}) = keytype(supertype(A))
Expand Down Expand Up @@ -449,19 +457,6 @@ copy(d::Dict) = Dict(d)

const AnyDict = Dict{Any,Any}

# TODO: this can probably be simplified using `eltype` as a THT (Tim Holy trait)
Dict{K,V}(kv::Tuple{Vararg{Tuple{K,V}}}) = Dict{K,V}(kv)
Dict{K }(kv::Tuple{Vararg{Tuple{K,Any}}}) = Dict{K,Any}(kv)
Dict{V }(kv::Tuple{Vararg{Tuple{Any,V}}}) = Dict{Any,V}(kv)
Dict{K,V}(kv::Tuple{Vararg{Pair{K,V}}}) = Dict{K,V}(kv)
Dict{K }(kv::Tuple{Vararg{Pair{K}}}) = Dict{K,Any}(kv)
Dict{V }(kv::Tuple{Vararg{Pair{TypeVar(:K),V}}}) = Dict{Any,V}(kv)
Dict( kv::Tuple{Vararg{Pair}}) = Dict{Any,Any}(kv)

Dict{K,V}(kv::AbstractArray{Tuple{K,V}}) = Dict{K,V}(kv)
Dict{K,V}(kv::AbstractArray{Pair{K,V}}) = Dict{K,V}(kv)
Dict{K,V}(kv::Associative{K,V}) = Dict{K,V}(kv)

Dict{K,V}(ps::Pair{K,V}...) = Dict{K,V}(ps)
Dict{K }(ps::Pair{K}...,) = Dict{K,Any}(ps)
Dict{V }(ps::Pair{TypeVar(:K),V}...,) = Dict{Any,V}(ps)
Expand All @@ -482,9 +477,27 @@ end

dict_with_eltype{K,V}(kv, ::Type{Tuple{K,V}}) = Dict{K,V}(kv)
dict_with_eltype{K,V}(kv, ::Type{Pair{K,V}}) = Dict{K,V}(kv)
dict_with_eltype(kv, t) = Dict{Any,Any}(kv)
dict_with_eltype(kv, t) = grow_to!(Dict{Union{},Union{}}(), kv)

# this is a special case due to (1) allowing both Pairs and Tuples as elements,
# and (2) Pair being invariant. a bit annoying.
function grow_to!{K,V}(dest::Associative{K,V}, itr, st = start(itr))
while !done(itr, st)
(k,v), st = next(itr, st)
if isa(k,K) && isa(v,V)
dest[k] = v
else
new = similar(dest, Pair{typejoin(K,typeof(k)), typejoin(V,typeof(v))})
copy!(new, dest)
new[k] = v
return grow_to!(new, itr, st)
end
end
return dest
end

similar{K,V}(d::Dict{K,V}) = Dict{K,V}()
similar{K,V}(d::Dict, ::Type{Pair{K,V}}) = Dict{K,V}()

# conversion between Dict types
function convert{K,V}(::Type{Dict{K,V}},d::Associative)
Expand Down
3 changes: 3 additions & 0 deletions base/essentials.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,9 @@ typealias Callable Union{Function,DataType}

const Bottom = Union{}

abstract AbstractSet{T}
abstract Associative{K,V}

# The real @inline macro is not available until after array.jl, so this
# internal macro splices the meta Expr directly into the function body.
macro _inline_meta()
Expand Down
Loading