Skip to content

Commit d4eb5cc

Browse files
yilinxiayilinxia
authored
Merge pull request #32 from data-exp-lab/fe-edge
Add Edges
2 parents c56c450 + 86177a6 commit d4eb5cc

23 files changed

+1675
-201
lines changed

‎backend/app/services/gexy_node_service.py‎

Lines changed: 45 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -61,30 +61,61 @@ def generate_gexf_nodes_for_topics(self, topics):
6161
topics_lower = [t.lower() for t in topics]
6262
placeholders = ",".join(["?"] * len(topics_lower))
6363

64+
# Debug: Check what columns actually exist in the repos table
65+
schema_query = "DESCRIBE repos"
66+
# try:
67+
# schema_result = self.con.execute(schema_query).fetchall()
68+
# # print("Repos table schema:")
69+
# # for col in schema_result:
70+
# # print(f" {col}")
71+
# except Exception as e:
72+
# print(f"Could not get schema: {e}")
73+
6474
query = f"""
65-
WITH repo_topics_agg AS (
66-
SELECT r.nameWithOwner,
67-
GROUP_CONCAT(t.topic, '|') as topics
75+
WITH matching_repos AS (
76+
SELECT DISTINCT r.nameWithOwner
6877
FROM repos r
6978
JOIN repo_topics t ON r.nameWithOwner = t.repo
7079
WHERE LOWER(t.topic) IN ({placeholders})
80+
),
81+
repo_topics_agg AS (
82+
SELECT
83+
r.nameWithOwner,
84+
GROUP_CONCAT(t.topic, '|') AS topics
85+
FROM repos r
86+
JOIN repo_topics t ON r.nameWithOwner = t.repo
87+
JOIN matching_repos mr ON r.nameWithOwner = mr.nameWithOwner
7188
GROUP BY r.nameWithOwner
7289
)
73-
SELECT DISTINCT r.nameWithOwner, r.stars, r.forks, r.watchers, r.isFork, r.isArchived,
74-
r.languageCount, r.pullRequests, r.issues, r.primaryLanguage, r.createdAt,
75-
r.license, rt.topics
90+
SELECT
91+
r.nameWithOwner,
92+
r.stars,
93+
r.forks,
94+
r.watchers,
95+
r.isArchived,
96+
r.languageCount,
97+
r.pullRequests,
98+
r.issues,
99+
r.primaryLanguage,
100+
r.createdAt,
101+
r.license,
102+
rt.topics
76103
FROM repos r
77-
JOIN repo_topics t ON r.nameWithOwner = t.repo
78-
JOIN repo_topics_agg rt ON r.nameWithOwner = rt.nameWithOwner
79-
WHERE LOWER(t.topic) IN ({placeholders})
104+
JOIN matching_repos mr ON r.nameWithOwner = mr.nameWithOwner
105+
JOIN repo_topics_agg rt ON r.nameWithOwner = rt.nameWithOwner;
80106
"""
81-
result = self.con.execute(query, topics_lower + topics_lower).fetchall()
107+
result = self.con.execute(query, topics_lower).fetchall()
108+
109+
# Debug: Print the first few rows to see what we're getting
110+
# if result:
111+
# print(f"First row sample: {result[0]}")
112+
# print(f"Number of results: {len(result)}")
113+
82114
columns = [
83115
"nameWithOwner",
84116
"stars",
85117
"forks",
86118
"watchers",
87-
"isFork",
88119
"isArchived",
89120
"languageCount",
90121
"pullRequests",
@@ -102,7 +133,6 @@ def generate_gexf_nodes_for_topics(self, topics):
102133
"stars": 0,
103134
"forks": 0,
104135
"watchers": 0,
105-
"isFork": False,
106136
"isArchived": False,
107137
"languageCount": 0,
108138
"pullRequests": 0,
@@ -119,7 +149,6 @@ def generate_gexf_nodes_for_topics(self, topics):
119149
'stars': {'type': 'integer'},
120150
'forks': {'type': 'integer'},
121151
'watchers': {'type': 'integer'},
122-
'isFork': {'type': 'boolean'},
123152
'isArchived': {'type': 'boolean'},
124153
'languageCount': {'type': 'integer'},
125154
'pullRequests': {'type': 'integer'},
@@ -157,6 +186,9 @@ def generate_gexf_nodes_for_topics(self, topics):
157186
elif col == "topics":
158187
# Store topics as a comma-separated string
159188
node_attrs[col] = val if val else default_values[col]
189+
elif col == "isArchived":
190+
# Ensure isArchived is always a boolean value
191+
node_attrs[col] = bool(val) if val is not None else False
160192
else:
161193
# Use default value if the value is None
162194
node_attrs[col] = default_values[col] if val is None else val

‎package-lock.json‎

Lines changed: 0 additions & 15 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎src/components/TopicRefiner.tsx‎

Lines changed: 111 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -98,13 +98,21 @@ export const TopicRefiner: FC<Omit<TopicRefinerProps, 'isLlmProcessing'>> = ({
9898
const [topicCounts, setTopicCounts] = useState<{ [key: string]: number }>({});
9999
const [uniqueReposCount, setUniqueReposCount] = useState<number>(0);
100100
const [showConfirmationModal, setShowConfirmationModal] = useState(false);
101+
const [isLoadingUniqueCount, setIsLoadingUniqueCount] = useState(false);
102+
const [uniqueCountError, setUniqueCountError] = useState<string | null>(null);
101103

102104
// Function to fetch unique repository count for all finalized topics
103105
const fetchUniqueReposCount = async (topics: string[]) => {
104106
if (topics.length === 0) {
105107
setUniqueReposCount(0);
108+
setIsLoadingUniqueCount(false);
109+
setUniqueCountError(null);
106110
return;
107111
}
112+
113+
setIsLoadingUniqueCount(true);
114+
setUniqueCountError(null);
115+
108116
try {
109117
const response = await fetch(API_ENDPOINTS.GET_UNIQUE_REPOS, {
110118
method: 'POST',
@@ -113,20 +121,54 @@ export const TopicRefiner: FC<Omit<TopicRefinerProps, 'isLlmProcessing'>> = ({
113121
},
114122
body: JSON.stringify({ topics }),
115123
});
124+
125+
if (!response.ok) {
126+
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
127+
}
128+
116129
const data = await response.json();
117130
if (data.success) {
118131
setUniqueReposCount(data.count);
132+
console.log('Unique repos count updated:', data.count);
133+
} else {
134+
throw new Error(data.message || 'Failed to get unique repos count');
119135
}
120136
} catch (error) {
121137
console.error('Error fetching unique repos count:', error);
138+
setUniqueCountError(error instanceof Error ? error.message : 'Unknown error');
139+
// Set a fallback count based on topic counts to ensure popup still works
140+
const fallbackCount = topics.reduce((total, topic) => {
141+
return total + (topicCounts[topic] || 0);
142+
}, 0);
143+
setUniqueReposCount(fallbackCount);
144+
} finally {
145+
setIsLoadingUniqueCount(false);
122146
}
123147
};
124148

125149
// Update unique repos count when finalized topics change
126150
useEffect(() => {
151+
console.log('Finalized topics changed, fetching unique repos count:', finalizedTopics);
127152
fetchUniqueReposCount(finalizedTopics);
128153
}, [finalizedTopics]);
129154

155+
// Also update when topicCounts change to ensure accuracy
156+
useEffect(() => {
157+
if (finalizedTopics.length > 0 && !isLoadingUniqueCount) {
158+
// Recalculate if we have new topic counts
159+
const hasAllCounts = finalizedTopics.every(topic => topicCounts[topic]);
160+
if (hasAllCounts) {
161+
const calculatedCount = finalizedTopics.reduce((total, topic) => {
162+
return total + (topicCounts[topic] || 0);
163+
}, 0);
164+
// Only update if it's significantly different to avoid unnecessary API calls
165+
if (Math.abs(calculatedCount - uniqueReposCount) > 100) {
166+
setUniqueReposCount(calculatedCount);
167+
}
168+
}
169+
}
170+
}, [topicCounts, finalizedTopics, isLoadingUniqueCount, uniqueReposCount]);
171+
130172
// Calculate total repositories for finalized topics using topicCounts
131173
const totalRepositories = useMemo(() => {
132174
return finalizedTopics.reduce((total, topic) => {
@@ -439,11 +481,28 @@ export const TopicRefiner: FC<Omit<TopicRefinerProps, 'isLlmProcessing'>> = ({
439481
}, []);
440482

441483
const handleSubmitFinalizedTopics = async () => {
484+
// Wait for unique count to be loaded if it's still loading
485+
if (isLoadingUniqueCount) {
486+
console.log('Waiting for unique count to load...');
487+
// Wait a bit for the count to load
488+
await new Promise(resolve => setTimeout(resolve, 500));
489+
}
490+
442491
// Add confirmation for large repository counts
443492
if (uniqueReposCount > 10000) {
493+
console.log('Showing confirmation modal for large dataset:', uniqueReposCount);
494+
setShowConfirmationModal(true);
495+
return;
496+
}
497+
498+
// If we have an error but still have a count, check if it's over threshold
499+
if (uniqueCountError && uniqueReposCount > 10000) {
500+
console.log('Showing confirmation modal despite error, count:', uniqueReposCount);
444501
setShowConfirmationModal(true);
445502
return;
446503
}
504+
505+
console.log('Proceeding with submission, unique count:', uniqueReposCount);
447506
await submitTopics();
448507
};
449508

@@ -845,17 +904,54 @@ export const TopicRefiner: FC<Omit<TopicRefinerProps, 'isLlmProcessing'>> = ({
845904
<div className="d-flex justify-content-end align-items-center mt-4 gap-3">
846905
<div className="text-muted" style={{ fontSize: '0.9rem' }}>
847906
<div>Total Topic Occurrences: <span className="fw-bold">{totalRepositories.toLocaleString()}</span></div>
848-
<div>Unique Repositories: <span className="fw-bold">{uniqueReposCount.toLocaleString()}</span></div>
907+
<div className="d-flex align-items-center gap-2">
908+
Unique Repositories:
909+
{isLoadingUniqueCount ? (
910+
<span className="d-flex align-items-center gap-1">
911+
<div className="spinner-border spinner-border-sm" role="status" style={{ width: "0.75rem", height: "0.75rem" }}>
912+
<span className="visually-hidden">Loading...</span>
913+
</div>
914+
<span className="fw-bold">Loading...</span>
915+
</span>
916+
) : uniqueCountError ? (
917+
<span className="fw-bold text-danger" title={uniqueCountError}>
918+
{uniqueReposCount.toLocaleString()} (Fallback)
919+
</span>
920+
) : (
921+
<span className="fw-bold">{uniqueReposCount.toLocaleString()}</span>
922+
)}
923+
</div>
924+
{uniqueCountError && (
925+
<div className="d-flex align-items-center gap-2 mt-1">
926+
<small className="text-danger">
927+
⚠️ Using fallback count due to API error
928+
</small>
929+
<button
930+
className="btn btn-sm btn-outline-secondary"
931+
onClick={() => fetchUniqueReposCount(finalizedTopics)}
932+
disabled={isLoadingUniqueCount}
933+
style={{ fontSize: '0.75rem', padding: '0.25rem 0.5rem' }}
934+
>
935+
{isLoadingUniqueCount ? (
936+
<div className="spinner-border spinner-border-sm" role="status" style={{ width: "0.75rem", height: "0.75rem" }}>
937+
<span className="visually-hidden">Loading...</span>
938+
</div>
939+
) : (
940+
'Retry'
941+
)}
942+
</button>
943+
</div>
944+
)}
849945
</div>
850946
<button
851947
className="btn d-flex align-items-center"
852948
onClick={handleSubmitFinalizedTopics}
853-
disabled={finalizedTopics.length === 0 || isSubmitting}
949+
disabled={finalizedTopics.length === 0 || isSubmitting || isLoadingUniqueCount}
854950
style={{
855951
color: 'white',
856952
backgroundColor: '#198754', // Bootstrap's success color
857953
borderColor: '#198754',
858-
opacity: finalizedTopics.length === 0 || isSubmitting ? 0.65 : 1
954+
opacity: finalizedTopics.length === 0 || isSubmitting || isLoadingUniqueCount ? 0.65 : 1
859955
}}
860956
>
861957
{isSubmitting ? (
@@ -1011,7 +1107,10 @@ export const TopicRefiner: FC<Omit<TopicRefinerProps, 'isLlmProcessing'>> = ({
10111107
<button
10121108
type="button"
10131109
className="btn-close"
1014-
onClick={() => setShowConfirmationModal(false)}
1110+
onClick={() => {
1111+
console.log('Confirmation modal closed by user');
1112+
setShowConfirmationModal(false);
1113+
}}
10151114
aria-label="Close"
10161115
></button>
10171116
</div>
@@ -1023,20 +1122,27 @@ export const TopicRefiner: FC<Omit<TopicRefinerProps, 'isLlmProcessing'>> = ({
10231122
<p className="mb-0">
10241123
This may take a while to process and could impact performance. Are you sure you want to continue?
10251124
</p>
1125+
<small className="d-block mt-2 text-muted">
1126+
Debug: Threshold is 10,000, current count: {uniqueReposCount}
1127+
</small>
10261128
</div>
10271129
</div>
10281130
<div className="modal-footer">
10291131
<button
10301132
type="button"
10311133
className="btn btn-secondary"
1032-
onClick={() => setShowConfirmationModal(false)}
1134+
onClick={() => {
1135+
console.log('Confirmation modal cancelled by user');
1136+
setShowConfirmationModal(false);
1137+
}}
10331138
>
10341139
Cancel
10351140
</button>
10361141
<button
10371142
type="button"
10381143
className="btn btn-danger"
10391144
onClick={async () => {
1145+
console.log('Confirmation modal confirmed by user, proceeding with submission');
10401146
setShowConfirmationModal(false);
10411147
await submitTopics();
10421148
}}

0 commit comments

Comments
 (0)