forked from rdicosmo/parmap
-
Notifications
You must be signed in to change notification settings - Fork 0
/
parmap.mli
198 lines (152 loc) · 10.2 KB
/
parmap.mli
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
(**************************************************************************)
(* ParMap: a simple library to perform Map computations on a multi-core *)
(* *)
(* Author(s): Marco Danelutto, Roberto Di Cosmo *)
(* *)
(* This library is free software: you can redistribute it and/or modify *)
(* it under the terms of the GNU Lesser General Public License as *)
(* published by the Free Software Foundation, either version 2 of the *)
(* License, or (at your option) any later version. A special linking *)
(* exception to the GNU Lesser General Public License applies to this *)
(* library, see the LICENSE file for more information. *)
(**************************************************************************)
(** Module [Parmap]: efficient parallel map, fold and mapfold on lists and
arrays on multicores.
All the primitives allow to control the granularity of the parallelism
via an optional parameter [chunksize]: if [chunksize] is omitted, the
input sequence is split evenly among the available cores; if [chunksize]
is specified, the input data is split in chunks of size [chunksize] and
dispatched to the available cores using an on demand strategy that
ensures automatic load balancing.
A specific primitive [array_float_parmap] is provided for fast operations on float arrays.
*)
(** {6 Setting and getting the default value for ncores } *)
val set_default_ncores : int -> unit
val get_default_ncores : unit -> int
(** {6 Sequence type, subsuming lists and arrays} *)
type 'a sequence = L of 'a list | A of 'a array;;
(** The [parmapfold], [parfold] and [parmap] generic functions, for efficiency reasons,
convert the input data into an array internally, so we provide the ['a sequence] type
to allow passing an array directly as input.
If you want to perform a parallel map operation on an array, use [array_parmap] or [array_float_parmap] instead.
*)
(** {6 Optional init and finalize functions} *)
(** The optional [init] (resp. [finalize]) function is called once by each child process just after creation
(resp. just before exit).
[init] and [finalize] both default to doing nothing.
[init i] takes the child rank [i] as parameter (first forked child has rank 0, next 1, etc.).
*)
(** {6 Parallel mapfold} *)
val parmapfold : ?init:(int -> unit) -> ?finalize:(unit -> unit) -> ?ncores:int -> ?chunksize:int -> ('a -> 'b) -> 'a sequence -> ('b-> 'c -> 'c) -> 'c -> ('c->'c->'c) -> 'c
(** [parmapfold ~ncores:n f (L l) op b concat ] computes [List.fold_right op (List.map f l) b]
by forking [n] processes on a multicore machine.
You need to provide the extra [concat] operator to combine the partial results of the
fold computed on each core. If 'b = 'c, then [concat] may be simply [op].
The order of computation in parallel changes w.r.t. sequential execution, so this
function is only correct if [op] and [concat] are associative and commutative.
If the optional [chunksize] parameter is specified,
the processes compute the result in an on-demand fashion
on blocks of size [chunksize].
[parmapfold ~ncores:n f (A a) op b concat ] computes [Array.fold_right op (Array.map f a) b]
*)
(** {6 Parallel fold} *)
val parfold: ?init:(int -> unit) -> ?finalize:(unit -> unit) -> ?ncores:int -> ?chunksize:int -> ('a -> 'b -> 'b) -> 'a sequence -> 'b -> ('b->'b->'b) -> 'b
(** [parfold ~ncores:n op (L l) b concat] computes [List.fold_right op l b]
by forking [n] processes on a multicore machine.
You need to provide the extra [concat] operator to combine the partial results of the
fold computed on each core. If 'b = 'c, then [concat] may be simply [op].
The order of computation in parallel changes w.r.t. sequential execution, so this
function is only correct if [op] and [concat] are associative and commutative.
If the optional [chunksize] parameter is specified,
the processes compute the result in an on-demand fashion
on blocks of size [chunksize].
[parfold ~ncores:n op (A a) b concat] similarly computes [Array.fold_right op a b].
*)
(** {6 Parallel map} *)
val parmap : ?init:(int -> unit) -> ?finalize:(unit -> unit) -> ?ncores:int -> ?chunksize:int -> ('a -> 'b) -> 'a sequence -> 'b list
(** [parmap ~ncores:n f (L l) ] computes [List.map f l]
by forking [n] processes on a multicore machine.
[parmap ~ncores:n f (A a) ] computes [Array.map f a]
by forking [n] processes on a multicore machine.
If the optional [chunksize] parameter is specified,
the processes compute the result in an on-demand fashion
on blocks of size [chunksize]; this provides automatic
load balancing for unbalanced computations, but the order
of the result is no longer guaranteed to be preserved. *)
(** {6 Parallel iteration} *)
val pariter : ?init:(int -> unit) -> ?finalize:(unit -> unit) -> ?ncores:int -> ?chunksize:int -> ('a -> unit) -> 'a sequence -> unit
(** [pariter ~ncores:n f (L l) ] computes [List.iter f l]
by forking [n] processes on a multicore machine.
[parmap ~ncores:n f (A a) ] computes [Array.iter f a]
by forking [n] processes on a multicore machine.
If the optional [chunksize] parameter is specified,
the processes perform the computation in an on-demand fashion
on blocks of size [chunksize]; this provides automatic
load balancing for unbalanced computations. *)
(** {6 Parallel mapfold, indexed} *)
val parmapifold : ?init:(int -> unit) -> ?finalize:(unit -> unit) -> ?ncores:int -> ?chunksize:int -> (int -> 'a -> 'b) -> 'a sequence -> ('b-> 'c -> 'c) -> 'c -> ('c->'c->'c) -> 'c
(** Like parmapfold, but the map function gets as an extra argument
the index of the mapped element *)
(** {6 Parallel map, indexed} *)
val parmapi : ?init:(int -> unit) -> ?finalize:(unit -> unit) -> ?ncores:int -> ?chunksize:int -> (int -> 'a -> 'b) -> 'a sequence -> 'b list
(** Like parmap, but the map function gets as an extra argument
the index of the mapped element *)
(** {6 Parallel iteration, indexed} *)
val pariteri : ?init:(int -> unit) -> ?finalize:(unit -> unit) -> ?ncores:int -> ?chunksize:int -> (int -> 'a -> unit) -> 'a sequence -> unit
(** Like pariter, but the iterated function gets as an extra argument
the index of the sequence element *)
(** {6 Parallel map on arrays} *)
val array_parmap : ?init:(int -> unit) -> ?finalize:(unit -> unit) -> ?ncores:int -> ?chunksize:int -> ('a -> 'b) -> 'a array -> 'b array
(** [array_parmap ~ncores:n f a ] computes [Array.map f a]
by forking [n] processes on a multicore machine.
If the optional [chunksize] parameter is specified,
the processes compute the result in an on-demand fashion
on blochs of size [chunksize]; this provides automatic
load balancing for unbalanced computations, but the order
of the result is no longer guaranteed to be preserved. *)
(** {6 Parallel map on arrays, indexed} *)
val array_parmapi : ?init:(int -> unit) -> ?finalize:(unit -> unit) -> ?ncores:int -> ?chunksize:int -> (int -> 'a -> 'b) -> 'a array -> 'b array
(** Like array_parmap, but the map function gets as an extra argument
the index of the mapped element *)
(** {6 Parallel map on float arrays } *)
exception WrongArraySize
type buf
val init_shared_buffer : float array -> buf
(** [init_shared_buffer a] creates a new memory mapped shared buffer big enough to hold a float array of the size of [a].
This buffer can be reused in a series of calls to [array_float_parmap], avoiding the cost of reallocating it each time. *)
val array_float_parmap : ?init:(int -> unit) -> ?finalize:(unit -> unit) -> ?ncores:int -> ?chunksize:int -> ?result: float array -> ?sharedbuffer: buf -> ('a -> float) -> 'a array -> float array
(** [array_float_parmap ~ncores:n f a ] computes [Array.map f a] by forking
[n] processes on a multicore machine, and preallocating the resulting
array as shared memory, which allows significantly more efficient
computation than calling the generic array_parmap function. If the
optional [chunksize] parameter is specified, the processes compute the
result in an on-demand fashion on blochs of size [chunksize]; this
provides automatic load balancing for unbalanced computations, *and* the
order of the result is still guaranteed to be preserved.
In case you already have at hand an array where to store the result, you
can squeeze out some more cpu cycles by passing it as optional parameter
[result]: this will avoid the creation of a result array, which can be
costly for very large data sets. Raises [WrongArraySize] if [result] is too
small to hold the data.
It is possible to share the same preallocated shared memory space across
calls, by initialising the space calling [init_shared_buffer a] and
passing the result as the optional [sharedbuffer] parameter to each
subsequent call to [array_float_parmap]. Raises WrongArraySize if
[sharedbuffer] is too small to hold the input data. *)
(** {6 Parallel map on float arrays, indexed } *)
val array_float_parmapi : ?init:(int -> unit) -> ?finalize:(unit -> unit) -> ?ncores:int -> ?chunksize:int -> ?result: float array -> ?sharedbuffer: buf -> (int -> 'a -> float) -> 'a array -> float array
(** Like array_float_parmap, but the map function gets as an extra argument
the index of the mapped element *)
(** {6 Debugging} *)
val debugging : bool -> unit
(** Enable or disable debugging code in the library; default: false *)
(** {6 Helper function for redirection of stdout and stderr} *)
val redirect : ?path:string -> id:int -> unit
(** Helper function that redirects stdout and stderr to files
located in the directory [path], carrying names of the shape
stdout.NNN and stderr.NNN where NNN is the [id] of the used core.
Useful when writing initialisation functions to be passed as
[init] argument to the parallel combinators.
The default value for [path] is /tmp/.parmap.PPPP with PPPP the
process id of the main program.
*)