{"payload":{"feedbackUrl":"https://github.com/orgs/community/discussions/53140","repo":{"id":811635561,"defaultBranch":"main","name":"finetuning","ownerLogin":"macrocosm-os","currentUserCanPush":false,"isFork":false,"isEmpty":false,"createdAt":"2024-06-07T01:58:39.000Z","ownerAvatar":"https://avatars.githubusercontent.com/u/163681588?v=4","public":true,"private":false,"isOrgOwned":true},"refInfo":{"name":"","listCacheKey":"v0:1726284509.0","currentOid":""},"activityList":{"items":[{"before":"52d78da71f3cf1bd6cff7a07d3968a27e1b9c076","after":"0a241d0b6d991451388bc8ccdea0952aa9bd510c","ref":"refs/heads/benchmark","pushedAt":"2024-09-16T20:21:05.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"RusticLuftig","name":null,"path":"/RusticLuftig","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/151780164?s=80&v=4"},"commit":{"message":"Run more benchmarks","shortMessageHtmlLink":"Run more benchmarks"}},{"before":"6efa57fcffd76bc7ebb6dd42169ed2c099a0efb6","after":"52d78da71f3cf1bd6cff7a07d3968a27e1b9c076","ref":"refs/heads/benchmark","pushedAt":"2024-09-15T16:51:25.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"RusticLuftig","name":null,"path":"/RusticLuftig","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/151780164?s=80&v=4"},"commit":{"message":"Fix comp selection","shortMessageHtmlLink":"Fix comp selection"}},{"before":"245b3956ce4f08ef3e3743a89af1d09d72c4dcbf","after":"6efa57fcffd76bc7ebb6dd42169ed2c099a0efb6","ref":"refs/heads/benchmark","pushedAt":"2024-09-15T16:49:06.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"RusticLuftig","name":null,"path":"/RusticLuftig","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/151780164?s=80&v=4"},"commit":{"message":"Only log the results","shortMessageHtmlLink":"Only log the results"}},{"before":"75965fde6922a008ebd859b86479c57aa8d5f026","after":"245b3956ce4f08ef3e3743a89af1d09d72c4dcbf","ref":"refs/heads/benchmark","pushedAt":"2024-09-15T16:41:33.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"RusticLuftig","name":null,"path":"/RusticLuftig","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/151780164?s=80&v=4"},"commit":{"message":"Reduce benchmarks run","shortMessageHtmlLink":"Reduce benchmarks run"}},{"before":"f51fbe9487747058b05e738746e6cb98be3991b4","after":"75965fde6922a008ebd859b86479c57aa8d5f026","ref":"refs/heads/benchmark","pushedAt":"2024-09-14T18:49:47.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"RusticLuftig","name":null,"path":"/RusticLuftig","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/151780164?s=80&v=4"},"commit":{"message":"Improve formatting","shortMessageHtmlLink":"Improve formatting"}},{"before":"7c65b1dfdb6578d6be446071526def666ba1c06d","after":"f51fbe9487747058b05e738746e6cb98be3991b4","ref":"refs/heads/benchmark","pushedAt":"2024-09-14T18:48:50.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"RusticLuftig","name":null,"path":"/RusticLuftig","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/151780164?s=80&v=4"},"commit":{"message":"Add a little more logging","shortMessageHtmlLink":"Add a little more logging"}},{"before":"18cc2964bd94e2d7b4713a0f451c651747c091c6","after":"7c65b1dfdb6578d6be446071526def666ba1c06d","ref":"refs/heads/benchmark","pushedAt":"2024-09-14T18:40:51.000Z","pushType":"force_push","commitsCount":0,"pusher":{"login":"RusticLuftig","name":null,"path":"/RusticLuftig","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/151780164?s=80&v=4"},"commit":{"message":"Add a benchmark tool\n\nAdd hf_dir flag\n\nFix env name for wandb access token\n\nTemp remove sleep\n\nRun on revision main, for now\n\nAdd some test code to run on an already benchmarked model\n\nLog results to file and compute LB scores\n\nDon't write the full results to json\n\nMake lm_eval optional\n\nConpute scores for all benchmarks\n\nLogin to HF\n\nTurn off samples\n\nTurn on debug logging\n\nUse a different model\n\nCreate file and only run a small bench for now\n\nAlso create eval store file\n\nFull leaderboard\n\nRun on the actual model\n\nEval comp 2\n\nIndexing is hard\n\nLoad model onto GPU\n\nRun all leaderboard benchmarks\n\nAdd logging to validation\n\nTry inputs[0]","shortMessageHtmlLink":"Add a benchmark tool"}},{"before":"17ef1ec46ca0bef7a284cbaf8fc9e8a9718470d5","after":"18cc2964bd94e2d7b4713a0f451c651747c091c6","ref":"refs/heads/benchmark","pushedAt":"2024-09-14T18:34:42.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"RusticLuftig","name":null,"path":"/RusticLuftig","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/151780164?s=80&v=4"},"commit":{"message":"Try inputs[0]","shortMessageHtmlLink":"Try inputs[0]"}},{"before":"b3312ade3696a83118f23b47c9b7063fbbddeccf","after":"17ef1ec46ca0bef7a284cbaf8fc9e8a9718470d5","ref":"refs/heads/benchmark","pushedAt":"2024-09-14T18:10:19.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"RusticLuftig","name":null,"path":"/RusticLuftig","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/151780164?s=80&v=4"},"commit":{"message":"Add logging to validation","shortMessageHtmlLink":"Add logging to validation"}},{"before":"d654eaf52156ecfaa025bf922fabc427c446531b","after":"b3312ade3696a83118f23b47c9b7063fbbddeccf","ref":"refs/heads/benchmark","pushedAt":"2024-09-14T17:53:00.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"RusticLuftig","name":null,"path":"/RusticLuftig","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/151780164?s=80&v=4"},"commit":{"message":"Run all leaderboard benchmarks","shortMessageHtmlLink":"Run all leaderboard benchmarks"}},{"before":"2ce5b2813f0854a2fd84e9725763388b484e5bb7","after":"d654eaf52156ecfaa025bf922fabc427c446531b","ref":"refs/heads/benchmark","pushedAt":"2024-09-14T17:19:35.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"RusticLuftig","name":null,"path":"/RusticLuftig","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/151780164?s=80&v=4"},"commit":{"message":"Load model onto GPU","shortMessageHtmlLink":"Load model onto GPU"}},{"before":"58870f40549c1ac4a79f6597d4c44bee0d665f61","after":"2ce5b2813f0854a2fd84e9725763388b484e5bb7","ref":"refs/heads/benchmark","pushedAt":"2024-09-14T16:41:29.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"RusticLuftig","name":null,"path":"/RusticLuftig","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/151780164?s=80&v=4"},"commit":{"message":"Indexing is hard","shortMessageHtmlLink":"Indexing is hard"}},{"before":"43b188ff76a2cb8d6741cb79b0735e15bdf589a7","after":"58870f40549c1ac4a79f6597d4c44bee0d665f61","ref":"refs/heads/benchmark","pushedAt":"2024-09-14T16:40:29.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"RusticLuftig","name":null,"path":"/RusticLuftig","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/151780164?s=80&v=4"},"commit":{"message":"Eval comp 2","shortMessageHtmlLink":"Eval comp 2"}},{"before":"c3d2fa979553f984a3f78d820c8cf10107f96a86","after":"43b188ff76a2cb8d6741cb79b0735e15bdf589a7","ref":"refs/heads/benchmark","pushedAt":"2024-09-14T16:36:34.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"RusticLuftig","name":null,"path":"/RusticLuftig","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/151780164?s=80&v=4"},"commit":{"message":"Run on the actual model","shortMessageHtmlLink":"Run on the actual model"}},{"before":"7e3661b16de897568e91bbcddbf6aeb37f2e90af","after":"c3d2fa979553f984a3f78d820c8cf10107f96a86","ref":"refs/heads/benchmark","pushedAt":"2024-09-14T04:40:25.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"RusticLuftig","name":null,"path":"/RusticLuftig","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/151780164?s=80&v=4"},"commit":{"message":"Full leaderboard","shortMessageHtmlLink":"Full leaderboard"}},{"before":"ed470c202428b1e38d119278bc4bd94aa7df1d6e","after":"7e3661b16de897568e91bbcddbf6aeb37f2e90af","ref":"refs/heads/benchmark","pushedAt":"2024-09-14T03:52:50.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"RusticLuftig","name":null,"path":"/RusticLuftig","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/151780164?s=80&v=4"},"commit":{"message":"Also create eval store file","shortMessageHtmlLink":"Also create eval store file"}},{"before":"cd8c1270efb40d71d258d2df28c9d279c2e84b79","after":"ed470c202428b1e38d119278bc4bd94aa7df1d6e","ref":"refs/heads/benchmark","pushedAt":"2024-09-14T03:51:56.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"RusticLuftig","name":null,"path":"/RusticLuftig","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/151780164?s=80&v=4"},"commit":{"message":"Create file and only run a small bench for now","shortMessageHtmlLink":"Create file and only run a small bench for now"}},{"before":"bb0d82e33cce9854dee3ddf658c7f14ebb3e70fd","after":"656c4a8daf47233e18078a349233638847bf3e49","ref":"refs/heads/main","pushedAt":"2024-09-14T03:27:45.000Z","pushType":"pr_merge","commitsCount":4,"pusher":{"login":"RusticLuftig","name":null,"path":"/RusticLuftig","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/151780164?s=80&v=4"},"commit":{"message":"Merge pull request #85 from macrocosm-os/dev\n\nRelease 2.1.1","shortMessageHtmlLink":"Merge pull request <a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"2525966266\" data-permission-text=\"Title is private\" data-url=\"https://github.com/macrocosm-os/finetuning/issues/85\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/macrocosm-os/finetuning/pull/85/hovercard\" href=\"https://github.com/macrocosm-os/finetuning/pull/85\">#85</a> from macrocosm-os/dev"}},{"before":"776c0b9e19b72e5ef8e76b9771e1bde504b30c24","after":"85fc26a8ba364986b121ad3860b68248a9995e9f","ref":"refs/heads/dev","pushedAt":"2024-09-14T03:25:43.000Z","pushType":"pr_merge","commitsCount":3,"pusher":{"login":"RusticLuftig","name":null,"path":"/RusticLuftig","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/151780164?s=80&v=4"},"commit":{"message":"Merge pull request #84 from macrocosm-os/wandb-hotfix\n\nWandb hotfix","shortMessageHtmlLink":"Merge pull request <a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"2525965662\" data-permission-text=\"Title is private\" data-url=\"https://github.com/macrocosm-os/finetuning/issues/84\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/macrocosm-os/finetuning/pull/84/hovercard\" href=\"https://github.com/macrocosm-os/finetuning/pull/84\">#84</a> from macrocosm-os/wandb-hotfix"}},{"before":"e4cd0fda0111d8da6ebfabb669fe1f192911d12f","after":"7d09b4855c546b0c6290cadb916f3cc1c6f8b6c4","ref":"refs/heads/wandb-hotfix","pushedAt":"2024-09-14T03:19:42.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"RusticLuftig","name":null,"path":"/RusticLuftig","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/151780164?s=80&v=4"},"commit":{"message":"Use a custom sampled history scan","shortMessageHtmlLink":"Use a custom sampled history scan"}},{"before":null,"after":"e4cd0fda0111d8da6ebfabb669fe1f192911d12f","ref":"refs/heads/wandb-hotfix","pushedAt":"2024-09-13T23:06:13.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"RusticLuftig","name":null,"path":"/RusticLuftig","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/151780164?s=80&v=4"},"commit":{"message":"Reduce samples and update wandb","shortMessageHtmlLink":"Reduce samples and update wandb"}},{"before":"920f4837680a0cbc9d74b28a3d5749bdfb835e1b","after":"bb0d82e33cce9854dee3ddf658c7f14ebb3e70fd","ref":"refs/heads/main","pushedAt":"2024-09-13T17:09:15.000Z","pushType":"pr_merge","commitsCount":4,"pusher":{"login":"RusticLuftig","name":null,"path":"/RusticLuftig","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/151780164?s=80&v=4"},"commit":{"message":"Merge pull request #83 from macrocosm-os/dev\n\nRelease 2.1.0","shortMessageHtmlLink":"Merge pull request <a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"2525296731\" data-permission-text=\"Title is private\" data-url=\"https://github.com/macrocosm-os/finetuning/issues/83\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/macrocosm-os/finetuning/pull/83/hovercard\" href=\"https://github.com/macrocosm-os/finetuning/pull/83\">#83</a> from macrocosm-os/dev"}},{"before":"6d8804ad142e646cf5d3fb657f20b2c5553930bd","after":"776c0b9e19b72e5ef8e76b9771e1bde504b30c24","ref":"refs/heads/dev","pushedAt":"2024-09-13T17:08:48.000Z","pushType":"pr_merge","commitsCount":3,"pusher":{"login":"RusticLuftig","name":null,"path":"/RusticLuftig","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/151780164?s=80&v=4"},"commit":{"message":"Merge pull request #82 from macrocosm-os/sunset-comp1\n\nSunset competition 1","shortMessageHtmlLink":"Merge pull request <a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"2525292827\" data-permission-text=\"Title is private\" data-url=\"https://github.com/macrocosm-os/finetuning/issues/82\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/macrocosm-os/finetuning/pull/82/hovercard\" href=\"https://github.com/macrocosm-os/finetuning/pull/82\">#82</a> from macrocosm-os/sunset-comp1"}},{"before":"7fdec0ef4b04002f9c148b6df5b2984be6da24ac","after":"6f82f9338ebbe33e092202eff22c29c018946adf","ref":"refs/heads/sunset-comp1","pushedAt":"2024-09-13T17:07:12.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"RusticLuftig","name":null,"path":"/RusticLuftig","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/151780164?s=80&v=4"},"commit":{"message":"Bump version","shortMessageHtmlLink":"Bump version"}},{"before":null,"after":"7fdec0ef4b04002f9c148b6df5b2984be6da24ac","ref":"refs/heads/sunset-comp1","pushedAt":"2024-09-13T16:49:21.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"RusticLuftig","name":null,"path":"/RusticLuftig","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/151780164?s=80&v=4"},"commit":{"message":"Sunset comp 1","shortMessageHtmlLink":"Sunset comp 1"}},{"before":"32baefcc728b9a5338ce20f1bedc3694f5bf5955","after":"cd8c1270efb40d71d258d2df28c9d279c2e84b79","ref":"refs/heads/benchmark","pushedAt":"2024-09-13T04:23:11.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"RusticLuftig","name":null,"path":"/RusticLuftig","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/151780164?s=80&v=4"},"commit":{"message":"Use a different model","shortMessageHtmlLink":"Use a different model"}},{"before":"67ce989b548bf893550d4626279fa9cb29115bad","after":"32baefcc728b9a5338ce20f1bedc3694f5bf5955","ref":"refs/heads/benchmark","pushedAt":"2024-09-12T16:21:19.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"RusticLuftig","name":null,"path":"/RusticLuftig","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/151780164?s=80&v=4"},"commit":{"message":"Turn on debug logging","shortMessageHtmlLink":"Turn on debug logging"}},{"before":"4f436deb12625469537a9bba0efac60c1532b960","after":"67ce989b548bf893550d4626279fa9cb29115bad","ref":"refs/heads/benchmark","pushedAt":"2024-09-12T16:15:04.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"RusticLuftig","name":null,"path":"/RusticLuftig","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/151780164?s=80&v=4"},"commit":{"message":"Turn off samples","shortMessageHtmlLink":"Turn off samples"}},{"before":"4d21245c9fe106576dd51809b6577e4e0d53df4a","after":"4f436deb12625469537a9bba0efac60c1532b960","ref":"refs/heads/benchmark","pushedAt":"2024-09-12T04:09:33.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"RusticLuftig","name":null,"path":"/RusticLuftig","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/151780164?s=80&v=4"},"commit":{"message":"Login to HF","shortMessageHtmlLink":"Login to HF"}},{"before":"1a820606c04e90635573d880361a36837ea509c8","after":"4d21245c9fe106576dd51809b6577e4e0d53df4a","ref":"refs/heads/benchmark","pushedAt":"2024-09-12T03:51:53.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"RusticLuftig","name":null,"path":"/RusticLuftig","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/151780164?s=80&v=4"},"commit":{"message":"Conpute scores for all benchmarks","shortMessageHtmlLink":"Conpute scores for all benchmarks"}}],"hasNextPage":true,"hasPreviousPage":false,"activityType":"all","actor":null,"timePeriod":"all","sort":"DESC","perPage":30,"cursor":"Y3Vyc29yOnYyOpK7MjAyNC0wOS0xNlQyMDoyMTowNS4wMDAwMDBazwAAAAS32RHm","startCursor":"Y3Vyc29yOnYyOpK7MjAyNC0wOS0xNlQyMDoyMTowNS4wMDAwMDBazwAAAAS32RHm","endCursor":"Y3Vyc29yOnYyOpK7MjAyNC0wOS0xMlQwMzo1MTo1My4wMDAwMDBazwAAAASz5wzE"}},"title":"Activity · macrocosm-os/finetuning"}