Skip to content
This repository was archived by the owner on Mar 7, 2025. It is now read-only.

Commit 494e8d7

Browse files
NunoSempereNunoSempere
committed
feat: Fix Good Judgment flow, start writting the Insight Parser.
Note that the Insight Prediction parser is in fact not complete
1 parent cb6f923 commit 494e8d7

File tree

3 files changed

+178
-62
lines changed

3 files changed

+178
-62
lines changed

‎src/backend/platforms/goodjudgmentopen.ts

Lines changed: 58 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -17,12 +17,19 @@ const annoyingPromptUrls = [
1717
"https://www.gjopen.com/questions/1779-are-there-any-forecasting-tips-tricks-and-experiences-you-would-like-to-share-and-or-discuss-with-your-fellow-forecasters",
1818
"https://www.gjopen.com/questions/2246-are-there-any-forecasting-tips-tricks-and-experiences-you-would-like-to-share-and-or-discuss-with-your-fellow-forecasters-2022-thread",
1919
"https://www.gjopen.com/questions/2237-what-forecasting-questions-should-we-ask-what-questions-would-you-like-to-forecast-on-gjopen",
20+
"https://www.gjopen.com/questions/2437-what-forecasting-questions-should-we-ask-what-questions-would-you-like-to-forecast-on-gjopen"
2021
];
2122
const DEBUG_MODE: "on" | "off" = "off"; // "on"
2223
const id = () => 0;
2324

2425
/* Support functions */
2526

27+
function cleanDescription(text: string) {
28+
let md = toMarkdown(text);
29+
let result = md.replaceAll("---", "-").replaceAll(" ", " ");
30+
return result;
31+
}
32+
2633
async function fetchPage(page: number, cookie: string) {
2734
const response: string = await axios({
2835
url: htmlEndPoint + page,
@@ -40,82 +47,68 @@ async function fetchStats(questionUrl: string, cookie: string) {
4047
url: questionUrl + "/stats",
4148
method: "GET",
4249
headers: {
50+
"Content-Type": "text/html",
4351
Cookie: cookie,
4452
Referer: questionUrl,
4553
},
4654
}).then((res) => res.data);
47-
//console.log(response)
4855

49-
// Is binary?
50-
let isbinary = response.includes("binary?":true");
56+
if (response.includes("Sign up or sign in to forecast")) {
57+
throw Error("Not logged in");
58+
}
59+
// Init
60+
let options: FullQuestionOption[] = [];
5161

52-
let options: FetchedQuestion["options"] = [];
53-
if (isbinary) {
54-
// Crowd percentage
55-
let htmlElements = response.split("\n");
56-
let h3Element = htmlElements.filter((str) => str.includes("<h3>"))[0];
57-
// console.log(h3Element)
58-
let crowdpercentage = h3Element.split(">")[1].split("<")[0];
59-
let probability = Number(crowdpercentage.replace("%", "")) / 100;
60-
options.push(
61-
{
62-
name: "Yes",
63-
probability: probability,
64-
type: "PROBABILITY",
65-
},
66-
{
67-
name: "No",
68-
probability: +(1 - probability).toFixed(2), // avoids floating point shenanigans
69-
type: "PROBABILITY",
70-
}
71-
);
72-
} else {
73-
let optionsHtmlElement = "<table" + response.split("tbody")[1] + "table>";
74-
let tablesAsJson = Tabletojson.convert(optionsHtmlElement);
75-
let firstTable = tablesAsJson[0];
76-
options = firstTable.map((element: any) => ({
77-
name: element["0"],
78-
probability: Number(element["1"].replace("%", "")) / 100,
62+
// Parse the embedded json
63+
let htmlElements = response.split("\n");
64+
let jsonLines = htmlElements.filter((element) =>
65+
element.includes("data-react-props")
66+
);
67+
let embeddedJsons = jsonLines.map((jsonLine, i) => {
68+
let innerJSONasHTML = jsonLine.split('data-react-props="')[1].split('"')[0];
69+
let json = JSON.parse(innerJSONasHTML.replaceAll("&quot;", '"'));
70+
return json;
71+
});
72+
let firstEmbeddedJson = embeddedJsons[0];
73+
let title = firstEmbeddedJson.question.name;
74+
let description = cleanDescription(firstEmbeddedJson.question.description);
75+
let comments_count = firstEmbeddedJson.question.comments_count;
76+
let numforecasters = firstEmbeddedJson.question.predictors_count;
77+
let numforecasts = firstEmbeddedJson.question.prediction_sets_count;
78+
let questionType = firstEmbeddedJson.question.type;
79+
if (
80+
questionType.includes("Binary") ||
81+
questionType.includes("NonExclusiveOpinionPoolQuestion") ||
82+
questionType.includes("Forecast::Question") ||
83+
!questionType.includes("Forecast::MultiTimePeriodQuestion")
84+
) {
85+
options = firstEmbeddedJson.question.answers.map((answer: any) => ({
86+
name: answer.name,
87+
probability: answer.normalized_probability,
7988
type: "PROBABILITY",
8089
}));
81-
//console.log(optionsHtmlElement)
82-
//console.log(options)
90+
if (options.length == 1 && options[0].name == "Yes") {
91+
let probabilityNo =
92+
options[0].probability > 1
93+
? 1 - options[0].probability / 100
94+
: 1 - options[0].probability;
95+
options.push({
96+
name: "No",
97+
probability: probabilityNo,
98+
type: "PROBABILITY",
99+
});
100+
}
83101
}
84-
85-
// Description
86-
let descriptionraw = response.split(
87-
`<div id="question-background" class="collapse smb">`
88-
)[1];
89-
let descriptionprocessed1 = descriptionraw.split(`</div>`)[0];
90-
let descriptionprocessed2 = toMarkdown(descriptionprocessed1);
91-
let descriptionprocessed3 = descriptionprocessed2
92-
.split("\n")
93-
.filter((string) => !string.includes("Confused? Check our"))
94-
.join("\n");
95-
let description = descriptionprocessed3;
96-
97-
// Number of forecasts
98-
let numforecasts = response
99-
.split("prediction_sets_count&quot;:")[1]
100-
.split(",")[0];
101-
//console.log(numforecasts)
102-
103-
// Number of predictors
104-
let numforecasters = response
105-
.split("predictors_count&quot;:")[1]
106-
.split(",")[0];
107-
//console.log(numpredictors)
108-
109102
let result = {
110-
description,
111-
options,
103+
description: description,
104+
options: options,
112105
qualityindicators: {
113106
numforecasts: Number(numforecasts),
114107
numforecasters: Number(numforecasters),
108+
comments_count: Number(comments_count),
115109
},
116-
// this mismatches the code below, and needs to be fixed, but I'm doing typescript conversion and don't want to touch any logic for now
117-
} as any;
118-
110+
};
111+
// console.log(JSON.stringify(result, null, 4));
119112
return result;
120113
}
121114

@@ -150,6 +143,7 @@ async function goodjudgmentopen_inner(cookie: string) {
150143
let results = [];
151144
let init = Date.now();
152145
// console.log("Downloading... This might take a couple of minutes. Results will be shown.")
146+
console.log("Page #1")
153147
while (!reachedEnd(response) && isSignedIn(response)) {
154148
let htmlLines = response.split("\n");
155149
DEBUG_MODE == "on" ? htmlLines.forEach((line) => console.log(line)) : id();
@@ -187,6 +181,8 @@ async function goodjudgmentopen_inner(cookie: string) {
187181
if (j % 30 == 0 || DEBUG_MODE == "on") {
188182
console.log(`Page #${i}`);
189183
console.log(question);
184+
}else{
185+
console.log(question.title)
190186
}
191187
// console.log(question)
192188
results.push(question);

‎src/backend/platforms/insight.ts

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
/* Imports */
2+
import axios from "axios";
3+
4+
import { FetchedQuestion, Platform } from ".";
5+
6+
/* Definitions */
7+
const platformName = "insight";
8+
const marketsEnpoint = "https://insightprediction.com/api/markets";
9+
const getMarketEndpoint = (id: number) => `https://insightprediction.com/api/markets/${id}`
10+
11+
/* Support functions */
12+
13+
async function fetchQuestionStats(bearer: string, marketId: number){
14+
const response = await axios({
15+
url: getMarketEndpoint(marketId),
16+
method: "GET",
17+
headers: {
18+
"Content-Type": "application/json",
19+
"Accept": "application/json",
20+
"Authorization": `Bearer ${bearer}`,
21+
},
22+
}).then((res) => res.data);
23+
// console.log(response)
24+
return response;
25+
}
26+
27+
async function fetchPage(bearer: string, pageNum: number) {
28+
const response = await axios({
29+
url: `${marketsEnpoint}?page=${pageNum}`,
30+
method: "GET",
31+
headers: {
32+
"Content-Type": "application/json",
33+
"Accept": "application/json",
34+
"Authorization": `Bearer ${bearer}`,
35+
},
36+
}).then((res) => res.data);
37+
// console.log(response)
38+
return response;
39+
}
40+
41+
async function fetchData(bearer: string){
42+
let pageNum = 1
43+
let reachedEnd = false
44+
let results = []
45+
while(!reachedEnd){
46+
let newPage = await fetchPage(bearer, pageNum)
47+
let newPageData = newPage.data
48+
49+
let marketsWithStats = newPageData.map(marketData => {
50+
let marketStats = fetchQuestionStats(bearer, marketData.id)
51+
return ({...marketStats, ...marketData})
52+
})
53+
54+
console.log(`Page = #${pageNum}`)
55+
// console.log(newPageData)
56+
console.log(marketsWithStats)
57+
results.push(...marketsWithStats)
58+
59+
let newPagination = newPage.meta.pagination
60+
if(newPagination.total_pages == pageNum ){
61+
reachedEnd = true
62+
}else{
63+
pageNum = pageNum + 1
64+
}
65+
}
66+
}
67+
68+
async function processPredictions(predictions: any[]) {
69+
let results = await predictions.map((prediction) => {
70+
const id = `${platformName}-${prediction.id}`;
71+
const probability = prediction.probability;
72+
const options: FetchedQuestion["options"] = [
73+
{
74+
name: "Yes",
75+
probability: probability,
76+
type: "PROBABILITY",
77+
},
78+
{
79+
name: "No",
80+
probability: 1 - probability,
81+
type: "PROBABILITY",
82+
},
83+
];
84+
const result: FetchedQuestion = {
85+
id,
86+
title: prediction.title,
87+
url: "https://example.com",
88+
description: prediction.description,
89+
options,
90+
qualityindicators: {
91+
// other: prediction.otherx,
92+
// indicators: prediction.indicatorx,
93+
},
94+
};
95+
return result;
96+
});
97+
return results; //resultsProcessed
98+
}
99+
100+
/* Body */
101+
102+
export const insight: Platform = {
103+
name: platformName,
104+
label: "Insight Prediction",
105+
color: "#ff0000",
106+
version: "v0",
107+
async fetcher() {
108+
let bearer = process.env.INSIGHT_BEARER;
109+
let pageNum = 1
110+
let data = await fetchData(bearer);
111+
console.log(data)
112+
let results = [] // await processPredictions(data); // somehow needed
113+
return results;
114+
},
115+
calculateStars(data) {
116+
return 2;
117+
},
118+
};

‎src/backend/platforms/registry.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import { goodjudgmentopen } from "./goodjudgmentopen";
77
import { guesstimate } from "./guesstimate";
88
import { Platform, PlatformConfig } from "./index";
99
import { infer } from "./infer";
10+
import { insight } from "./insight";
1011
import { kalshi } from "./kalshi";
1112
import { manifold } from "./manifold";
1213
import { metaculus } from "./metaculus";
@@ -28,6 +29,7 @@ export const getPlatforms = (): Platform<string>[] => {
2829
goodjudgmentopen,
2930
guesstimate,
3031
infer,
32+
insight,
3133
kalshi,
3234
manifold,
3335
metaculus,

0 commit comments

Comments
 (0)