Skip to content

Commit

Permalink
Create benchmarks (#255)
Browse files Browse the repository at this point in the history
  • Loading branch information
Airyzz authored Jun 3, 2024
1 parent a793884 commit 2a289fd
Show file tree
Hide file tree
Showing 15 changed files with 528 additions and 10 deletions.
76 changes: 76 additions & 0 deletions .github/workflows/benchmark.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
on:
merge_group:
pull_request:
workflow_dispatch:

name: benchmark

permissions:
contents: write
deployments: write

env:
PROJECT_PATH: commet

jobs:
benchmark:
name: benchmark
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v2
with:
submodules: 'true'

- name: Setup Flutter
uses: subosito/[email protected]
with:
flutter-version: '3.19.5'
channel: 'stable'

- name: Setup Dependencies
run: |
sudo apt-get update -y
sudo apt-get install -y ninja-build libgtk-3-dev libolm3 libmpv-dev mpv ffmpeg libmimalloc-dev
- name: Configure Flutter
run: flutter config --enable-linux-desktop

- name: Code Generation
run: |
cd $PROJECT_PATH
dart run scripts/codegen.dart
- name: Benchmark
run: |
cd $PROJECT_PATH
export DISPLAY=:99
sudo Xvfb -ac :99 -screen 0 1920x1080x24 > /dev/null 2>&1 &
ffmpeg -f x11grab -video_size 1920x1080 -framerate 15 -i :99 -vcodec libx264 -preset ultrafast -qp 0 -nostdin -loglevel quiet -pix_fmt yuv444p video.mkv &
flutter drive -d linux --driver=test_driver/benchmark_driver.dart --target=integration_test/benchmark/benchmark.dart --profile
INTEGRATION_TEST_EXIT_CODE=$?
sleep 1
kill $(pgrep ffmpeg)
exit $INTEGRATION_TEST_EXIT_CODE
- name: Upload Artifact
uses: actions/upload-artifact@v3
with:
name: result.mkv
path: commet/video.mkv

- name: Store benchmark result - separate results repo
uses: benchmark-action/github-action-benchmark@v1
with:
name: Benchmark
tool: 'customSmallerIsBetter'
output-file-path: commet/build/customSmallerIsBetter.json
github-token: ${{ secrets.BENCHMARK_ACTION_BOT_TOKEN }}
auto-push: true
# Show alert with commit comment on detecting possible performance regression
alert-threshold: '350%'
comment-on-alert: true
fail-on-alert: false
summary-always: true
gh-repository: 'github.com/commetchat/commet_benchmark_results'
8 changes: 8 additions & 0 deletions commet/integration_test/benchmark/benchmark.dart
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
import 'package:integration_test/integration_test.dart';
import 'benchmarks/timeline_viewer_benchmark.dart' as timeline;

main() {
IntegrationTestWidgetsFlutterBinding.ensureInitialized();

timeline.main();
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
import 'package:commet/diagnostic/benchmark_values.dart';
import 'package:commet/ui/pages/developer/benchmarks/benchmark_utils.dart';
import 'package:commet/ui/pages/developer/benchmarks/timeline_viewer_benchmark.dart';
import 'package:flutter/material.dart';
import 'package:flutter_test/flutter_test.dart';
import 'package:integration_test/integration_test.dart';
import 'package:tiamat/config/style/theme_dark.dart';

void main() {
final binding = IntegrationTestWidgetsFlutterBinding.ensureInitialized();

testWidgets('Timeline Viewer Test', (tester) async {
// Build our app and trigger a frame.
await tester.pumpWidget(MaterialApp(
theme: ThemeDark.theme,
home: const Scaffold(
body: BenchmarkTimelineViewer(),
),
));

await tester.pump(const Duration(seconds: 1));

final listFinder = find.byType(Scrollable);
final itemFinder = find.text(finalEventMessage);

var reportKey = 'TimelineViewer Scrolling';

await binding.traceAction(
() async {
// Scroll until the item to be found appears.
await tester.scrollUntilVisible(
itemFinder,
50.0,
maxScrolls: 10000,
scrollable: listFinder,
);
},
reportKey: reportKey,
);

binding.reportData?[reportKey]["extra_values"] = [
{
"name": "$reportKey - Timeline Event Build Count",
"value": BenchmarkValues.numTimelineEventsBuilt,
"unit": "Builds",
},
{
"name": "$reportKey - Timeline Event Message Body Build Count",
"value": BenchmarkValues.numTimelineMessageBodyBuilt,
"unit": "Builds",
},
{
"name": "$reportKey - Timeline Event Message Reply Body Build Count",
"value": BenchmarkValues.numTimelineReplyBodyBuilt,
"unit": "Builds",
},
{
"name": "$reportKey - Timeline Event Message Url Preview Build Count",
"value": BenchmarkValues.numTimelineUrlPreviewBuilt,
"unit": "Builds",
}
];
});
}
8 changes: 4 additions & 4 deletions commet/lib/client/matrix/matrix_client.dart
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ import 'package:olm/olm.dart' as olm;

class MatrixClient extends Client {
late matrix.Client _matrixClient;
late final List<Component<MatrixClient>> _components;
late final List<Component<MatrixClient>> componentsInternal;

Future? firstSync;

Expand Down Expand Up @@ -72,7 +72,7 @@ class MatrixClient extends Client {
_id = identifier;
_matrixClient = _createMatrixClient(identifier);
_matrixClient.onSync.stream.listen(onMatrixClientSync);
_components = ComponentRegistry.getMatrixComponents(this);
componentsInternal = ComponentRegistry.getMatrixComponents(this);
}

static String hash(String name) {
Expand Down Expand Up @@ -471,7 +471,7 @@ class MatrixClient extends Client {

@override
T? getComponent<T extends Component>() {
for (var component in _components) {
for (var component in componentsInternal) {
if (component is T) return component as T;
}

Expand All @@ -481,7 +481,7 @@ class MatrixClient extends Client {
@override
List<T>? getAllComponents<T extends Component<Client>>() {
List<T> components = List.empty(growable: true);
for (var component in _components) {
for (var component in componentsInternal) {
if (component is T) {
components.add(component as T);
}
Expand Down
14 changes: 12 additions & 2 deletions commet/lib/client/matrix/matrix_timeline.dart
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,18 @@ class MatrixTimeline extends Timeline {
MatrixTimeline(
Client client,
Room room,
matrix.Room matrixRoom,
) {
matrix.Room matrixRoom, {
matrix.Timeline? initialTimeline,
}) {
events = List.empty(growable: true);
_matrixRoom = matrixRoom;
this.client = client;
this.room = room;
_matrixTimeline = initialTimeline;

if (_matrixTimeline != null) {
convertAllTimelineEvents();
}
}

Future<void> initTimeline() async {
Expand All @@ -35,6 +41,10 @@ class MatrixTimeline extends Timeline {

// This could maybe make load times realllly slow if we have a ton of stuff in the cache?
// Might be better to only convert as many as we would need to display immediately and then convert the rest on demand
convertAllTimelineEvents();
}

void convertAllTimelineEvents() {
for (int i = 0; i < _matrixTimeline!.events.length; i++) {
var converted = MatrixTimelineEvent(
_matrixTimeline!.events[i], _matrixTimeline!.room.client,
Expand Down
6 changes: 6 additions & 0 deletions commet/lib/diagnostic/benchmark_values.dart
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
class BenchmarkValues {
static int numTimelineEventsBuilt = 0;
static int numTimelineMessageBodyBuilt = 0;
static int numTimelineReplyBodyBuilt = 0;
static int numTimelineUrlPreviewBuilt = 0;
}
51 changes: 51 additions & 0 deletions commet/lib/diagnostic/mocks/matrix_client_component_mocks.dart
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
import 'package:commet/client/components/url_preview/url_preview_component.dart';
import 'package:commet/client/matrix/matrix_client.dart';
import 'package:commet/client/room.dart';
import 'package:commet/client/timeline.dart';
import 'package:flutter/material.dart';

extension MatrixClientComponentReplacement on MatrixClient {
void mockComponents() {
componentsInternal.removeWhere((element) => element is UrlPreviewComponent);
componentsInternal.add(MockMatrixUrlPreviewComponent(this));
}
}

class MockMatrixUrlPreviewComponent
implements UrlPreviewComponent<MatrixClient> {
@override
MatrixClient client;

MockMatrixUrlPreviewComponent(this.client);

var imageIndex = 0;

@override
Future<UrlPreviewData?> getPreview(Room room, TimelineEvent event) async {
await Future.delayed(const Duration(seconds: 1));

var image = [
"assets/images/app_icon/app_icon_filled.png",
"assets/images/app_icon/app_icon_rounded.png",
"assets/images/app_icon/app_icon_transparent_cropped.png",
"assets/images/app_icon/app_icon_transparent.png"
][imageIndex % 4];

imageIndex += 1;

return UrlPreviewData(Uri.parse("https://example.com"),
siteName: "Example",
description: "Example description",
image: AssetImage(image));
}

@override
bool shouldGetPreviewData(Room room, TimelineEvent event) {
return true;
}

@override
UrlPreviewData? getCachedPreview(Room room, TimelineEvent event) {
return null;
}
}
3 changes: 3 additions & 0 deletions commet/lib/ui/molecules/message.dart
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import 'package:commet/client/components/emoticon/emoticon.dart';
import 'package:commet/client/components/url_preview/url_preview_component.dart';
import 'package:commet/config/build_config.dart';
import 'package:commet/diagnostic/benchmark_values.dart';
import 'package:commet/ui/atoms/emoji_reaction.dart';
import 'package:commet/ui/molecules/url_preview_widget.dart';
import 'package:commet/utils/link_utils.dart';
Expand Down Expand Up @@ -144,6 +145,7 @@ class _MessageState extends State<Message> {
}

Widget urlPreviews() {
BenchmarkValues.numTimelineUrlPreviewBuilt += 1;
return Padding(
padding: const EdgeInsets.fromLTRB(0, 0, 40, 0),
child: UrlPreviewWidget(
Expand Down Expand Up @@ -183,6 +185,7 @@ class _MessageState extends State<Message> {
}

Widget replyText() {
BenchmarkValues.numTimelineReplyBodyBuilt += 1;
return SizedBox(
child: Row(
crossAxisAlignment: CrossAxisAlignment.baseline,
Expand Down
6 changes: 6 additions & 0 deletions commet/lib/ui/molecules/timeline_event.dart
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import 'package:commet/client/components/url_preview/url_preview_component.dart';
import 'package:commet/config/build_config.dart';
import 'package:commet/diagnostic/benchmark_values.dart';
import 'package:commet/main.dart';
import 'package:commet/ui/atoms/generic_room_event.dart';
import 'package:commet/ui/molecules/message.dart';
Expand Down Expand Up @@ -46,6 +47,8 @@ class TimelineEventView extends StatefulWidget {
final bool canDeleteEvent;
final Function(Emoticon emote)? onReactionTapped;

static int timelineEventBuildsCount = 0;

@override
State<TimelineEventView> createState() => _TimelineEventState();
}
Expand Down Expand Up @@ -212,6 +215,8 @@ class _TimelineEventState extends State<TimelineEventView> {

@override
Widget build(BuildContext context) {
BenchmarkValues.numTimelineEventsBuilt += 1;

return eventToWidget(widget.event) ?? Container();
}

Expand Down Expand Up @@ -328,6 +333,7 @@ class _TimelineEventState extends State<TimelineEventView> {
}

Widget buildBody() {
BenchmarkValues.numTimelineMessageBodyBuilt += 1;
switch (widget.event.type) {
case EventType.message:
return buildMessageBody();
Expand Down
Loading

0 comments on commit 2a289fd

Please sign in to comment.