Merge pull request #637 from amock/fixr18

Update to build with OTP 18

master/eunit.config

@@ -1,4 +1,6 @@
-{eunit_compile_opts, [debug_info, fail_on_warning]}.
+{eunit_compile_opts, [debug_info, fail_on_warning,
+                      {platform_define, "^[0-9]+", namespaced_types},
+                      {platform_define, "^(18|19|2\d+|\d{3,})", time_correction}]}.
 {cover_enabled, true}.
 {deps,
  [{mochiweb, "",

master/rebar.config

@@ -1,7 +1,8 @@
 {erl_opts,
  [debug_info, warn_missing_spec, warnings_as_errors, nowarn_deprecated_type,
   {parse_transform, lager_transform},
-  {platform_define, "^[0-9]+", namespaced_types}
+  {platform_define, "^[0-9]+", namespaced_types},
+  {platform_define, "^(18|19|2\d+|\d{3,})", time_correction}
 ]}.
 {deps,
 [{lager, "2.0.3",

master/src/ddfs/ddfs_gc.erl

@@ -49,28 +49,28 @@ start_gc(Root) ->
 
 -spec initial_wait(timeout()) -> ok.
 initial_wait(InitialWait) ->
-    Start = now(),
+    Start = disco_util:timestamp(),
     receive
         {From, _Req} ->
             From ! {ok, init_wait},
-            Wait = timer:now_diff(now(), Start) div 1000,
+            Wait = timer:now_diff(disco_util:timestamp(), Start) div 1000,
             initial_wait(InitialWait - Wait);
         Other ->
             lager:error("GC: got unexpected msg ~p", [Other]),
-            Wait = timer:now_diff(now(), Start) div 1000,
+            Wait = timer:now_diff(disco_util:timestamp(), Start) div 1000,
             initial_wait(InitialWait - Wait)
     after InitialWait ->
             ok
     end.
 
 -spec start_gc(string(), ets:tab(), non_neg_integer()) -> no_return().
 start_gc(Root, DeletedAges, GCMaxDuration) ->
-    Start = now(),
+    Start = disco_util:timestamp(),
     case ddfs_gc_main:start_link(Root, DeletedAges) of
         {ok, Gc} ->
             start_gc_wait(Gc, GCMaxDuration),
             % timer:now_diff() returns microseconds.
-            Wait = timer:now_diff(now(), Start) div 1000,
+            Wait = timer:now_diff(disco_util:timestamp(), Start) div 1000,
             % Wait until the next scheduled gc run slot.
             Idle = ?GC_INTERVAL - (Wait rem ?GC_INTERVAL),
             idle(Idle);
@@ -82,41 +82,41 @@ start_gc(Root, DeletedAges, GCMaxDuration) ->
 
 -spec idle(timeout()) -> ok.
 idle(Timeout) ->
-    Start = now(),
+    Start = disco_util:timestamp(),
     receive
         {From, status} ->
             From ! {ok, not_running},
-            Wait = Timeout - timer:now_diff(now(), Start) div 1000,
+            Wait = Timeout - timer:now_diff(disco_util:timestamp(), Start) div 1000,
             idle(Wait);
         {From, start} ->
             From ! ok;
         _Other ->
-            Wait = Timeout - timer:now_diff(now(), Start) div 1000,
+            Wait = Timeout - timer:now_diff(disco_util:timestamp(), Start) div 1000,
             idle(Wait)
     after Timeout ->
             ok
     end.
 
 -spec start_gc_wait(pid(), timeout()) -> ok.
 start_gc_wait(Pid, Interval) ->
-    Start = now(),
+    Start = disco_util:timestamp(),
     receive
         {'EXIT', Pid, shutdown} ->
             lager:info("GC terminated.");
         {'EXIT', Pid, Reason} ->
             lager:error("GC: exited with ~p", [Reason]);
         {'EXIT', Other, Reason} ->
             lager:error("GC: got unexpected exit of ~p: ~p", [Other, Reason]),
-            start_gc_wait(Pid, Interval - (timer:now_diff(now(), Start) div 1000));
+            start_gc_wait(Pid, Interval - (timer:now_diff(disco_util:timestamp(), Start) div 1000));
         {From, status} when is_pid(From) ->
             ddfs_gc_main:gc_status(Pid, From),
-            start_gc_wait(Pid, Interval - (timer:now_diff(now(), Start) div 1000));
+            start_gc_wait(Pid, Interval - (timer:now_diff(disco_util:timestamp(), Start) div 1000));
         {From, start} when is_pid(From) ->
             From ! ok,
-            start_gc_wait(Pid, Interval - (timer:now_diff(now(), Start) div 1000));
+            start_gc_wait(Pid, Interval - (timer:now_diff(disco_util:timestamp(), Start) div 1000));
         Other ->
             lager:error("GC: got unexpected msg ~p", [Other]),
-            start_gc_wait(Pid, Interval - (timer:now_diff(now(), Start) div 1000))
+            start_gc_wait(Pid, Interval - (timer:now_diff(disco_util:timestamp(), Start) div 1000))
     after Interval ->
             lager:error("GC: completion timed out"),
             exit(Pid, force_timeout)

master/src/ddfs/ddfs_gc_main.erl

@@ -207,7 +207,7 @@
           phase    = start            :: phase(),
           gc_peers = gb_trees:empty() :: node_map(),
 
-          last_response_time = now()          :: erlang:timestamp(),
+          last_response_time = disco_util:timestamp()          :: disco_util:timestamp(),
           progress_timer    = undefined       :: 'undefined' | timer:tref(),
           gc_stats          = init_gc_stats() :: gc_run_stats(),
 
@@ -290,7 +290,7 @@ init({Root, DeletedAges}) ->
     % gc_blob_map: {Key :: {object_name(), node()},
     %               State :: 'pending' | 'missing' | check_blob_result()}
     % gc_tag_map:  {Key :: tagname(),
-    %               Id  :: erlang:timestamp()}
+    %               Id  :: disco_util:timestamp()}
     _ = ets:new(gc_blob_map, [named_table, set, private]),
     _ = ets:new(gc_tag_map, [named_table, set, private]),
 
@@ -303,7 +303,7 @@ init({Root, DeletedAges}) ->
 -spec handle_call(is_orphan_msg(), from(), state()) -> gs_reply(boolean() | unknown);
                  (dbg_state_msg(), from(), state()) -> gs_reply(state()).
 handle_call({is_orphan, Type, ObjName, Node, Vol}, _, S) ->
-    S1 = S#state{last_response_time = now()},
+    S1 = S#state{last_response_time = disco_util:timestamp()},
     {reply, check_is_orphan(S, Type, ObjName, Node, Vol), S1};
 
 handle_call(dbg_get_state, _, S) ->
@@ -331,7 +331,7 @@ handle_cast(start, #state{phase = start} = S) ->
     case get_all_tags() of
         {ok, Tags, OkNodes} ->
             Phase = build_map,
-            Peers = start_gc_peers(OkNodes, self(), now(), Phase),
+            Peers = start_gc_peers(OkNodes, self(), disco_util:timestamp(), Phase),
             % We iterate over the tags by messaging ourselves, so that
             % we keep processing our message queue, which would
             % otherwise fill up when we process very large numbers of
@@ -357,7 +357,7 @@ handle_cast({retry_node, Node},
                 Overused -> overused;
                 _ ->        normal
            end,
-    Pid = ddfs_gc_node:start_gc_node(Node, self(), now(), Phase, Mode),
+    Pid = ddfs_gc_node:start_gc_node(Node, self(), disco_util:timestamp(), Phase, Mode),
     Peers = update_peer(GCPeers, Node, Pid),
     case Phase of
         P when P =:= build_map; P =:= map_wait ->
@@ -414,7 +414,7 @@ handle_cast({build_map, []}, #state{phase = build_map} = S) ->
                  {ok, ProgressTimer} =
                      timer:send_after(?GC_PROGRESS_INTERVAL, check_progress),
                  S#state{phase = map_wait,
-                         last_response_time = now(),
+                         last_response_time = disco_util:timestamp(),
                          progress_timer = ProgressTimer,
                          num_pending_reqs = Pending}
          end,
@@ -478,7 +478,7 @@ handle_cast({gc_done, Node, GCNodeStats}, #state{phase = gc,
          end,
     {noreply, S1#state{pending_nodes = NewPending,
                        gc_stats = NewStats,
-                       last_response_time = now()}};
+                       last_response_time = disco_util:timestamp()}};
 
 handle_cast({rr_blob, '$end_of_table'},
             #state{phase = rr_blobs, rr_pid = RR, rr_reqs = RReqs} = S) ->
@@ -550,13 +550,13 @@ handle_info({check_blob_result, LocalObj, Status},
                  start_gc_phase(S);
              _ ->
                  S#state{num_pending_reqs = Pending,
-                         last_response_time = now()}
+                         last_response_time = disco_util:timestamp()}
          end,
     {noreply, S1};
 
 handle_info(check_progress, #state{phase = Phase, last_response_time = LRT} = S)
   when Phase =:= build_map; Phase =:= map_wait; Phase =:= gc ->
-    Since = timer:now_diff(now(), LRT),
+    Since = timer:now_diff(disco_util:timestamp(), LRT),
     case Since < ?GC_PROGRESS_INTERVAL of
         true ->
             % We have been making forward progress, restart the
@@ -638,7 +638,7 @@ schedule_retry(Node) ->
               end),
     ok.
 
--spec start_gc_peers([node()], pid(), erlang:timestamp(), phase()) -> node_map().
+-spec start_gc_peers([node()], pid(), disco_util:timestamp(), phase()) -> node_map().
 start_gc_peers(Nodes, Self, Now, Phase) ->
     lists:foldl(
       fun(N, Peers) ->
@@ -900,7 +900,7 @@ start_gc_phase(#state{gc_peers = Peers, nodestats = NodeStats} = S) ->
     S#state{num_pending_reqs  = 0,
             pending_nodes = gb_sets:from_list(gb_trees:keys(Peers)),
             phase = gc,
-            last_response_time = now(),
+            last_response_time = disco_util:timestamp(),
             overused_nodes = OverusedNodes,
             underused_nodes = UnderusedNodes,
             most_overused_node = MostOverused}.
@@ -977,7 +977,7 @@ rebalance(Overused, BL, NodeStats) ->
                           case ddfs_rebalance:is_balanced(Balanced, Threshold, DiskSpace) of
                           true ->
                               % The node has passed the threshold for how much of
-                              % its diskspace that can be selected to replicate 
+                              % its diskspace that can be selected to replicate
                               % for balancing.
                               Stats;
                           false ->
@@ -1146,7 +1146,7 @@ obj_stats(Type, {{KeptF, KeptB}, {DelF, DelB}}) ->
 -spec process_deleted([object_name()], ets:tab()) -> ok.
 process_deleted(Tags, Ages) ->
     lager:info("GC: Pruning +deleted"),
-    Now = now(),
+    Now = disco_util:timestamp(),
 
     % Let's start with the current list of deleted tags
     {ok, Deleted} = ddfs_master:tag_operation(get_tagnames,

master/src/ddfs/ddfs_master.erl

@@ -45,7 +45,7 @@
                 read_blacklist    = []                :: [node()],
                 gc_blacklist      = []                :: [node()],
                 safe_gc_blacklist = gb_sets:empty()   :: disco_gbset(tagname()),
-                gc_stats          = none              :: none | {gc_stats(), erlang:timestamp()}}).
+                gc_stats          = none              :: none | {gc_stats(), disco_util:timestamp()}}).
 -type state() :: #state{}.
 -type replyto() :: {pid(), reference()}.
 
@@ -90,7 +90,7 @@ gc_blacklist() ->
 gc_blacklist(Nodes) ->
     gen_server:cast(?MODULE, {gc_blacklist, Nodes}).
 
--spec gc_stats() -> {ok, none | {gc_stats(), erlang:timestamp()}} | {error, term()}.
+-spec gc_stats() -> {ok, none | {gc_stats(), disco_util:timestamp()}} | {error, term()}.
 gc_stats() ->
     gen_server:call(?MODULE, gc_stats).
 
@@ -181,7 +181,7 @@ init(_Args) ->
                  (gc_blacklist, from(), state()) ->
                          gs_reply({ok, [node()]});
                  (gc_stats, from(), state()) ->
-                         gs_reply({ok, gc_stats(), erlang:timestamp()});
+                         gs_reply({ok, gc_stats(), disco_util:timestamp()});
                  (choose_write_nodes_msg(), from(), state()) ->
                          gs_reply({ok, [node()]});
                  (new_blob_msg(), from(), state()) ->
@@ -261,7 +261,7 @@ handle_cast({safe_gc_blacklist, SafeBlacklist}, #state{gc_blacklist = BL} = S) -
     {noreply, S#state{safe_gc_blacklist = SBL}};
 
 handle_cast({update_gc_stats, Stats}, S) ->
-    {noreply, S#state{gc_stats = {Stats, now()}}};
+    {noreply, S#state{gc_stats = {Stats, disco_util:timestamp()}}};
 
 handle_cast({update_tag_cache, TagCache}, S) ->
     {noreply, S#state{tag_cache = {true, TagCache}}};

master/src/ddfs/ddfs_tag_util.erl

@@ -103,7 +103,7 @@ decode_tagcontent(TagData) ->
 
 -spec update_tagcontent(tagname(), tagcontent()) -> tagcontent().
 update_tagcontent(TagName, Tag) ->
-    Tag#tagcontent{id = ddfs_util:pack_objname(TagName, now()),
+    Tag#tagcontent{id = ddfs_util:pack_objname(TagName, disco_util:timestamp()),
                    last_modified = ddfs_util:format_timestamp()}.
 
 -spec update_tagcontent(tagname(), attrib(), _, _, token()) ->

master/src/ddfs/ddfs_util.erl

@@ -47,22 +47,22 @@ startswith(B, Prefix) ->
     Head =:= Prefix.
 
 -spec timestamp() -> string().
-timestamp() -> timestamp(now()).
+timestamp() -> timestamp(disco_util:timestamp()).
 
--spec timestamp(erlang:timestamp()) -> string().
+-spec timestamp(disco_util:timestamp()) -> string().
 timestamp({X0, X1, X2}) ->
     lists:flatten([to_hex(X0), $-, to_hex(X1), $-, to_hex(X2)]).
 
--spec timestamp_to_time(nonempty_string()) -> erlang:timestamp().
+-spec timestamp_to_time(nonempty_string()) -> disco_util:timestamp().
 timestamp_to_time(T) ->
     list_to_tuple([erlang:list_to_integer(X, 16)
                    || X <- string:tokens(lists:flatten(T), "-")]).
 
--spec pack_objname(tagname(), erlang:timestamp()) -> tagid().
+-spec pack_objname(tagname(), disco_util:timestamp()) -> tagid().
 pack_objname(Name, T) ->
     list_to_binary([Name, "$", timestamp(T)]).
 
--spec unpack_objname(tagid() | string()) -> {binary(), erlang:timestamp()}.
+-spec unpack_objname(tagid() | string()) -> {binary(), disco_util:timestamp()}.
 unpack_objname(Obj) when is_binary(Obj) ->
     unpack_objname(binary_to_list(Obj));
 unpack_objname(Obj) ->
@@ -88,7 +88,7 @@ ensure_dir(Dir) ->
 
 -spec format_timestamp() -> binary().
 format_timestamp() ->
-    {Date, Time} = calendar:now_to_local_time(now()),
+    {Date, Time} = calendar:now_to_local_time(disco_util:timestamp()),
     DateStr = io_lib:fwrite("~w/~.2.0w/~.2.0w ", tuple_to_list(Date)),
     TimeStr = io_lib:fwrite("~.2.0w:~.2.0w:~.2.0w", tuple_to_list(Time)),
     list_to_binary([DateStr, TimeStr]).

master/src/disco.erl

@@ -207,9 +207,9 @@ format_time(Ms, Second, Minute, Hour) ->
     lists:flatten(io_lib:format("~B:~2.10.0B:~2.10.0B.~3.10.0B",
                                 [Hour, Minute, Second, Ms])).
 
--spec format_time_since(erlang:timestamp()) -> nonempty_string().
+-spec format_time_since(disco_util:timestamp()) -> nonempty_string().
 format_time_since(Time) ->
-    format_time(timer:now_diff(now(), Time)).
+    format_time(timer:now_diff(disco_util:timestamp(), Time)).
 
 -spec make_dir(file:filename()) -> {ok, file:filename()} | {error, _}.
 make_dir(Dir) ->

master/src/disco_server.erl

@@ -16,7 +16,7 @@
 -include("disco.hrl").
 -include("pipeline.hrl").
 
--type connection_status() :: undefined | {up | down, erlang:timestamp()}.
+-type connection_status() :: undefined | {up | down, disco_util:timestamp()}.
 
 -record(dnode, {host :: host(),
                 node_mon :: pid(),
@@ -31,7 +31,7 @@
 
 -record(state, {workers = gb_trees:empty() :: disco_gbtree(pid(), {host(), task()}),
                 nodes   = gb_trees:empty() :: disco_gbtree(host(), dnode()),
-                purged  = gb_trees:empty() :: disco_gbtree(binary(), erlang:timestamp()),
+                purged  = gb_trees:empty() :: disco_gbtree(binary(), disco_util:timestamp()),
                 jobpack_queue :: pid(),
 
                 % The below are only used in cluster-in-a-box mode.
@@ -353,10 +353,10 @@ do_connection_status(Node, Status, #state{nodes = Nodes} = S) ->
     UpdatedNodes =
         case gb_trees:lookup(Node, Nodes) of
             {value, N} when Status =:= up ->
-                N1 = N#dnode{connection_status = {up, now()}},
+                N1 = N#dnode{connection_status = {up, disco_util:timestamp()}},
                 gb_trees:update(Node, N1, Nodes);
             {value, N} when Status =:= down ->
-                N1 = N#dnode{connection_status = {down, now()}},
+                N1 = N#dnode{connection_status = {down, disco_util:timestamp()}},
                 gb_trees:update(Node, N1, Nodes);
             _ -> Nodes
         end,
@@ -409,7 +409,7 @@ do_update_config_table(Config, Blacklist, GCBlacklist,
                               {#dnode{host = Host,
                                       node_mon = node_mon:start_link(Host, NodePorts),
                                       manual_blacklist = lists:member(Host, Blacklist),
-                                      connection_status = {down, now()},
+                                      connection_status = {down, disco_util:timestamp()},
                                       slots = Slots,
                                       num_running = 0,
                                       stats_ok = 0,
@@ -487,7 +487,7 @@ do_purge_job(JobName, #state{purged = Purged} = S) ->
             true ->
                 Purged;
             false ->
-                gb_trees:insert(Key, now(), Purged)
+                gb_trees:insert(Key, disco_util:timestamp(), Purged)
         end,
     S#state{purged = NPurged}.
 
@@ -557,7 +557,7 @@ do_get_nodeinfo(#state{nodes = Nodes}) ->
 
 -spec do_get_purged(state()) -> {{ok, [binary()]}, state()}.
 do_get_purged(#state{purged = Purged} = S) ->
-    Now = now(),
+    Now = disco_util:timestamp(),
     NPurgedList =
         [{Job, TStamp} || {Job, TStamp} <- gb_trees:to_list(Purged),
                           timer:now_diff(Now, TStamp) < ?PURGE_TIMEOUT * 1000],

master/src/disco_util.erl

@@ -1,6 +1,16 @@
 -module(disco_util).
 -export([choose_random/1, choose_random/2, groupby/2,
-         format_timestamp/1]).
+         format_timestamp/1, timestamp/0]).
+
+-ifdef(time_correction).
+-spec timestamp() -> erlang:timestamp().
+timestamp() ->
+    erlang:timestamp().
+-else.
+-spec timestamp() -> erlang:timestamp().
+timestamp() ->
+    now().
+-endif.
 
 -spec format_timestamp(erlang:timestamp()) -> binary().
 format_timestamp(TimeStamp) ->