Skip to content

Commit f137e76

Browse files
AklakanAklakan
committed
Added monitoring sources and channels
1 parent 6c23e55 commit f137e76

15 files changed

+2366
-67
lines changed

‎aksw-commons-io-parent/aksw-commons-io-buffers/src/main/java/org/aksw/commons/io/input/ChannelMonitor.java

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,18 @@
11
package org.aksw.commons.io.input;
22

3+
import java.io.IOException;
4+
import java.io.OutputStream;
5+
import java.io.OutputStreamWriter;
6+
import java.util.Map.Entry;
7+
38
import com.google.common.collect.Range;
49
import com.google.common.collect.RangeMap;
510
import com.google.common.collect.RangeSet;
611
import com.google.common.collect.TreeRangeMap;
712
import com.google.common.collect.TreeRangeSet;
13+
import com.google.gson.Gson;
14+
import com.google.gson.GsonBuilder;
15+
import com.google.gson.stream.JsonWriter;
816

917
public class ChannelMonitor {
1018
public class RangeTracker {
@@ -84,4 +92,20 @@ public synchronized void submitReadStats(long readStartPos, long readEndPos, int
8492
});
8593
}
8694
}
95+
96+
public void dumpJson(OutputStream out) throws IOException {
97+
Gson gson = new GsonBuilder().setPrettyPrinting().create();
98+
try (JsonWriter w = gson.newJsonWriter(new OutputStreamWriter(out))) {
99+
w.beginArray();
100+
for (Entry<Range<Long>, RangeTracker> e : trackedReads.asMapOfRanges().entrySet()) {
101+
w.beginObject();
102+
w.name("offset");
103+
w.value(e.getKey().lowerEndpoint());
104+
w.name("length");
105+
w.value(e.getValue().getMaxReadLength());
106+
w.endObject();
107+
}
108+
w.endArray();
109+
}
110+
}
87111
}

‎aksw-commons-io-parent/aksw-commons-io-buffers/src/main/java/org/aksw/commons/io/input/ChannelMonitor2.java

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
package org.aksw.commons.io.input;
2+
3+
import java.io.IOException;
4+
import java.io.OutputStream;
5+
import java.io.OutputStreamWriter;
6+
import java.util.Map.Entry;
7+
import java.util.NavigableMap;
8+
import java.util.TreeMap;
9+
import java.util.concurrent.atomic.AtomicLong;
10+
11+
import com.google.gson.Gson;
12+
import com.google.gson.GsonBuilder;
13+
import com.google.gson.stream.JsonWriter;
14+
15+
public class ChannelMonitor2 {
16+
public class RangeTracker {
17+
protected long totalDurationNanos;
18+
// protected Set<Integer> readLengths;
19+
protected int minReadLength;
20+
protected int maxReadLength;
21+
protected long totalReadLength;
22+
protected long readCount;
23+
24+
public RangeTracker() {
25+
this(Integer.MAX_VALUE, 0, 0, 0, 0);
26+
}
27+
28+
public RangeTracker(int readLength, long totalDurationNanos) {
29+
this(readLength, readLength, readLength, totalDurationNanos, 1);
30+
}
31+
32+
public RangeTracker(int minReadLength, int maxReadLength, long totalReadLength, long totalDurationNanos, long readCount) {
33+
super();
34+
this.totalDurationNanos = totalDurationNanos;
35+
this.minReadLength = minReadLength;
36+
this.maxReadLength = maxReadLength;
37+
this.totalReadLength = totalReadLength;
38+
this.readCount = readCount;
39+
}
40+
41+
public long getTotalDurationNanos() {
42+
return totalDurationNanos;
43+
}
44+
45+
public int getMinReadLength() {
46+
return minReadLength;
47+
}
48+
49+
public long getTotalReadLength() {
50+
return totalReadLength;
51+
}
52+
53+
public int getMaxReadLength() {
54+
return maxReadLength;
55+
}
56+
57+
public long getReadCount() {
58+
return readCount;
59+
}
60+
61+
public void add(RangeTracker contrib) {
62+
this.totalDurationNanos += contrib.totalDurationNanos;
63+
this.maxReadLength = Math.max(this.maxReadLength, contrib.maxReadLength);
64+
this.minReadLength = this.minReadLength == -1
65+
? contrib.minReadLength
66+
: Math.min(this.minReadLength, contrib.minReadLength);
67+
this.totalReadLength += contrib.totalReadLength;
68+
++this.readCount;
69+
}
70+
71+
@Override
72+
protected RangeTracker clone() {
73+
return new RangeTracker(minReadLength, maxReadLength, totalReadLength, totalDurationNanos, readCount);
74+
}
75+
}
76+
77+
protected volatile NavigableMap<Long, RangeTracker> trackedReads = new TreeMap<>();
78+
protected AtomicLong readCounter = new AtomicLong();
79+
protected AtomicLong readAmount = new AtomicLong();
80+
81+
public void addReadAmount(long readAmount) {
82+
this.readAmount.addAndGet(readAmount);
83+
}
84+
85+
public void incReadCounter() {
86+
this.readCounter.addAndGet(1);
87+
}
88+
89+
public long getReadCounter() {
90+
return readCounter.get();
91+
}
92+
93+
public long getReadAmount() {
94+
return readAmount.get();
95+
}
96+
97+
public NavigableMap<Long, RangeTracker> getTrackedReads() {
98+
return trackedReads;
99+
}
100+
101+
public synchronized void submitReadStats(long offset, long readStartPos, long readEndPos, int readLength, long durationNanos) {
102+
if (readLength > 0) { // Skip lengths that are <= 0
103+
RangeTracker tracker = trackedReads.computeIfAbsent(offset, o -> new RangeTracker());
104+
RangeTracker contrib = new RangeTracker(readLength, durationNanos);
105+
tracker.add(contrib);
106+
}
107+
}
108+
109+
public void dumpJson(OutputStream out) throws IOException {
110+
Gson gson = new GsonBuilder().setPrettyPrinting().create();
111+
try (JsonWriter w = gson.newJsonWriter(new OutputStreamWriter(out))) {
112+
w.beginArray();
113+
for (Entry<Long, RangeTracker> e : getTrackedReads().entrySet()) {
114+
w.beginObject();
115+
w.name("offset");
116+
w.value(e.getKey());
117+
w.name("length");
118+
w.value(e.getValue().getTotalReadLength());
119+
w.endObject();
120+
}
121+
w.endArray();
122+
}
123+
}
124+
125+
}

‎aksw-commons-io-parent/aksw-commons-io-buffers/src/main/java/org/aksw/commons/io/input/SeekableReadableChannelSource.java

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22

33
import java.io.IOException;
44

5+
import com.google.common.collect.Range;
6+
57
public interface SeekableReadableChannelSource<A>
68
extends ReadableChannelSource<A>
79
{
@@ -15,6 +17,16 @@ default SeekableReadableChannel<A> newReadableChannel(long offset) throws IOExce
1517
return result;
1618
}
1719

20+
@Override
21+
default SeekableReadableChannel<A> newReadableChannel(long start, long end) throws IOException {
22+
throw new UnsupportedOperationException();
23+
}
24+
25+
@Override
26+
default SeekableReadableChannel<A> newReadableChannel(Range<Long> range) throws IOException {
27+
throw new UnsupportedOperationException();
28+
}
29+
1830

1931
// return newReadableChannel(0l);
2032
// }

‎aksw-commons-io-parent/aksw-commons-io-buffers/src/main/java/org/aksw/commons/io/input/SeekableReadableChannelWithMonitor.java

Lines changed: 20 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,15 @@
66
public class SeekableReadableChannelWithMonitor<A, X extends SeekableReadableChannel<A>>
77
extends SeekableReadableChannelDecoratorBase<A, X>
88
{
9-
protected ChannelMonitor monitor;
9+
protected ChannelMonitor2 monitor;
1010

11-
public SeekableReadableChannelWithMonitor(X delegate, ChannelMonitor monitor) {
11+
protected long cachedPos;
12+
protected long relativeStart;
13+
protected long readLength;
14+
15+
protected volatile long readCounter = 0;
16+
17+
public SeekableReadableChannelWithMonitor(X delegate, ChannelMonitor2 monitor) {
1218
super(delegate);
1319
this.monitor = Objects.requireNonNull(monitor);
1420
}
@@ -17,18 +23,24 @@ public SeekableReadableChannelWithMonitor(X delegate, ChannelMonitor monitor) {
1723
public int read(A array, int position, int length) throws IOException {
1824
// Include positioning in the time so that long times may be discovered
1925
long startTimestamp = System.nanoTime();
20-
long startPos = super.position();
2126
int result = super.read(array, position, length);
22-
long endPos = super.position();
23-
long endTimestamp = System.nanoTime();
24-
long duration = endTimestamp - startTimestamp;
25-
monitor.submitReadStats(startPos, endPos, result, duration);
26-
27+
if (result > 0) {
28+
// long endPos = super.position();
29+
long endTimestamp = System.nanoTime();
30+
long duration = endTimestamp - startTimestamp;
31+
long nextStart = relativeStart + result;
32+
// System.out.println(String.format("Read #%d: pos %d len: %d", ++readCounter, cachedPos, result)) ;
33+
monitor.incReadCounter();
34+
monitor.addReadAmount(result);
35+
// monitor.submitReadStats(cachedPos, relativeStart, nextStart, result, duration);
36+
relativeStart = nextStart;
37+
}
2738
return result;
2839
}
2940

3041
@Override
3142
public void position(long pos) throws IOException {
43+
this.cachedPos = pos;
3244
super.position(pos);
3345
}
3446
}

‎aksw-commons-io-parent/aksw-commons-io-buffers/src/main/java/org/aksw/commons/io/input/SeekableReadableSourceWithMonitor.java

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
package org.aksw.commons.io.input;
2+
3+
import java.util.Objects;
4+
5+
public class SeekableReadableSourceWithMonitor<A, X extends SeekableReadableChannelSource<A>>
6+
extends SeekableReadableSourceWrapperBase<A, X>
7+
{
8+
protected ChannelMonitor2 channelMonitor;
9+
10+
public SeekableReadableSourceWithMonitor(X delegate) {
11+
this(delegate, new ChannelMonitor2());
12+
}
13+
14+
public ChannelMonitor2 getChannelMonitor() {
15+
return channelMonitor;
16+
}
17+
18+
public SeekableReadableSourceWithMonitor(X delegate,
19+
ChannelMonitor2 channelMonitor) {
20+
super(Objects.requireNonNull(delegate));
21+
this.channelMonitor = Objects.requireNonNull(channelMonitor);
22+
}
23+
24+
@Override
25+
protected SeekableReadableChannel<A> wrap(SeekableReadableChannel<A> delegate) {
26+
return new SeekableReadableChannelWithMonitor<>(delegate, channelMonitor);
27+
}
28+
}

‎aksw-commons-io-parent/aksw-commons-io-buffers/src/main/java/org/aksw/commons/io/input/SeekableReadableSourceWrapperBase.java

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
package org.aksw.commons.io.input;
2+
3+
import java.io.IOException;
4+
5+
import org.aksw.commons.io.buffer.array.ArrayOps;
6+
7+
import com.google.common.collect.Range;
8+
9+
public abstract class SeekableReadableSourceWrapperBase<A, X extends SeekableReadableChannelSource<A>>
10+
implements SeekableReadableChannelSource<A>
11+
{
12+
protected X delegate;
13+
14+
public SeekableReadableSourceWrapperBase(X delegate) {
15+
super();
16+
this.delegate = delegate;
17+
}
18+
19+
public X getDelegate() {
20+
return delegate;
21+
}
22+
23+
@Override
24+
public long size() throws IOException {
25+
return getDelegate().size();
26+
}
27+
28+
@Override
29+
public ArrayOps<A> getArrayOps() {
30+
return getDelegate().getArrayOps();
31+
}
32+
33+
@Override
34+
public SeekableReadableChannel<A> newReadableChannel() throws IOException {
35+
return wrap(getDelegate().newReadableChannel());
36+
}
37+
38+
@Override
39+
public SeekableReadableChannel<A> newReadableChannel(long start) throws IOException {
40+
SeekableReadableChannel<A> result = newReadableChannel();
41+
result.position(start);
42+
return result;
43+
// return wrap(getDelegate().newReadableChannel(start));
44+
}
45+
46+
@Override
47+
public SeekableReadableChannel<A> newReadableChannel(long start, long end) throws IOException {
48+
SeekableReadableChannel<A> result = newReadableChannel();
49+
result.position(start);
50+
return result;
51+
// return wrap(getDelegate().newReadableChannel(start, end));
52+
}
53+
54+
@Override
55+
public SeekableReadableChannel<A> newReadableChannel(Range<Long> range) throws IOException {
56+
throw new UnsupportedOperationException();
57+
// SeekableReadableChannel<A> result = newReadableChannel();
58+
// result.position(range.low);
59+
// return result;
60+
// return wrap(getDelegate().newReadableChannel(range));
61+
}
62+
63+
protected abstract SeekableReadableChannel<A> wrap(SeekableReadableChannel<A> delegate);
64+
}

‎aksw-commons-io-parent/aksw-commons-io-buffers/src/main/java/org/aksw/commons/io/input/SeekableReadableSourceWrapperWithMonitor.java

Lines changed: 0 additions & 57 deletions
This file was deleted.

0 commit comments

Comments
 (0)