Skip to content

Commit d90332d

Browse files
abravermabraverm
committed
refactor all
1 parent 8060ade commit d90332d

File tree

4 files changed

+124
-86
lines changed

4 files changed

+124
-86
lines changed

‎packages/backend/src/gitlab.ts

Lines changed: 98 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import { PrismaClient } from "@sourcebot/db";
77
import { processPromiseResults, throwIfAnyFailed } from "./connectionUtils.js";
88
import * as Sentry from "@sentry/node";
99
import { env } from "./env.js";
10+
import { log } from "console";
1011

1112
const logger = createLogger('gitlab');
1213
export const GITLAB_CLOUD_HOSTNAME = "gitlab.com";
@@ -45,15 +46,27 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig, o
4546
if (config.all === true) {
4647
if (hostname !== GITLAB_CLOUD_HOSTNAME) {
4748
try {
48-
logger.debug(`Fetching all projects visible in ${config.url}...`);
49-
const { durationMs, data: _projects } = await measure(async () => {
50-
const fetchFn = () => api.Projects.all({
51-
perPage: 100,
52-
});
53-
return fetchWithRetry(fetchFn, `all projects in ${config.url}`, logger);
49+
// Fetch all groups
50+
logger.debug(`Fetching all groups visible in ${config.url}...`);
51+
const { durationMs: groupsDuration, data: _groups } = await measure(async () => {
52+
const fetchFn = () => api.Groups.all({ perPage: 100, allAvailable: true });
53+
return fetchWithRetry(fetchFn, `all groups in ${config.url}`, logger);
54+
});
55+
logger.debug(`Found ${_groups.length} groups in ${groupsDuration}ms.`);
56+
57+
config.groups = _groups.map(g => g.full_path);
58+
59+
logger.debug(`Found these groups: ${config.groups.join('\n')}`);
60+
61+
// Fetch all users - too much for sourcebot/gitlab
62+
logger.debug(`Fetching all users visible in ${config.url}...`);
63+
const { durationMs: usersDuration, data: _users } = await measure(async () => {
64+
const fetchFn = () => api.Users.all({ perPage: 100, withoutProjects: false });
65+
return fetchWithRetry(fetchFn, `all users in ${config.url}`, logger);
5466
});
55-
logger.debug(`Found ${_projects.length} projects in ${durationMs}ms.`);
56-
allRepos = allRepos.concat(_projects);
67+
logger.debug(`Found ${_users.length} users in ${usersDuration}ms.`);
68+
69+
config.users = _users.map(u => u.username);
5770
} catch (e) {
5871
Sentry.captureException(e);
5972
logger.error(`Failed to fetch all projects visible in ${config.url}.`, e);
@@ -65,76 +78,96 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig, o
6578
}
6679

6780
if (config.groups) {
68-
const results = await Promise.allSettled(config.groups.map(async (group) => {
69-
try {
70-
logger.debug(`Fetching project info for group ${group}...`);
71-
const { durationMs, data } = await measure(async () => {
72-
const fetchFn = () => api.Groups.allProjects(group, {
73-
perPage: 100,
74-
includeSubgroups: true
81+
const batchSize = 10;
82+
const allResults = [];
83+
84+
// Process groups in batches of 10
85+
for (let i = 0; i < config.groups.length; i += batchSize) {
86+
const batch = config.groups.slice(i, i + batchSize);
87+
logger.debug(`Processing batch ${i/batchSize + 1} of ${Math.ceil(config.groups.length/batchSize)} (${batch.length} groups)`);
88+
89+
const batchResults = await Promise.allSettled(batch.map(async (group) => {
90+
try {
91+
logger.debug(`Fetching project info for group ${group}...`);
92+
const { durationMs, data } = await measure(async () => {
93+
const fetchFn = () => api.Groups.allProjects(group, {
94+
perPage: 100,
95+
includeSubgroups: true
96+
});
97+
return fetchWithRetry(fetchFn, `group ${group}`, logger);
7598
});
76-
return fetchWithRetry(fetchFn, `group ${group}`, logger);
77-
});
78-
logger.debug(`Found ${data.length} projects in group ${group} in ${durationMs}ms.`);
79-
return {
80-
type: 'valid' as const,
81-
data
82-
};
83-
} catch (e: any) {
84-
Sentry.captureException(e);
85-
logger.error(`Failed to fetch projects for group ${group}.`, e);
86-
87-
const status = e?.cause?.response?.status;
88-
if (status === 404) {
89-
logger.error(`Group ${group} not found or no access`);
99+
logger.debug(`Found ${data.length} projects in group ${group} in ${durationMs}ms.`);
90100
return {
91-
type: 'notFound' as const,
92-
value: group
101+
type: 'valid' as const,
102+
data
93103
};
94-
}
95-
throw e;
96-
}
97-
}));
104+
} catch (e: any) {
105+
Sentry.captureException(e);
106+
logger.error(`Failed to fetch projects for group ${group}.`, e);
98107

99-
throwIfAnyFailed(results);
100-
const { validItems: validRepos, notFoundItems: notFoundOrgs } = processPromiseResults(results);
108+
const status = e?.cause?.response?.status;
109+
if (status === 404) {
110+
logger.error(`Group ${group} not found or no access`);
111+
return {
112+
type: 'notFound' as const,
113+
value: group
114+
};
115+
}
116+
throw e;
117+
}
118+
}));
119+
allResults.push(...batchResults);
120+
}
121+
const { validItems: validRepos, notFoundItems: notFoundOrgs } = processPromiseResults(allResults);
101122
allRepos = allRepos.concat(validRepos);
102123
notFound.orgs = notFoundOrgs;
124+
logger.debug(`Found ${validRepos.length} valid repositories in groups.`);
125+
logger.debug(`Not found groups: ${notFoundOrgs.join(', ')}`);
126+
logger.debug(`These repositories will be downloaded: ${allRepos.map(repo => repo.path_with_namespace).join('\n')}`);
103127
}
104128

105129
if (config.users) {
106-
const results = await Promise.allSettled(config.users.map(async (user) => {
107-
try {
108-
logger.debug(`Fetching project info for user ${user}...`);
109-
const { durationMs, data } = await measure(async () => {
110-
const fetchFn = () => api.Users.allProjects(user, {
111-
perPage: 100,
130+
const batchSize = 10;
131+
const allResults = [];
132+
133+
// Process users in batches of 10
134+
for (let i = 0; i < config.users.length; i += batchSize) {
135+
const batch = config.users.slice(i, i + batchSize);
136+
logger.debug(`Processing batch ${i/batchSize + 1} of ${Math.ceil(config.users.length/batchSize)} (${batch.length} users)`);
137+
138+
const batchResults = await Promise.allSettled(batch.map(async (user) => {
139+
try {
140+
logger.debug(`Fetching project info for user ${user}...`);
141+
const { durationMs, data } = await measure(async () => {
142+
const fetchFn = () => api.Users.allProjects(user, {
143+
perPage: 100,
144+
});
145+
return fetchWithRetry(fetchFn, `user ${user}`, logger);
112146
});
113-
return fetchWithRetry(fetchFn, `user ${user}`, logger);
114-
});
115-
logger.debug(`Found ${data.length} projects owned by user ${user} in ${durationMs}ms.`);
116-
return {
117-
type: 'valid' as const,
118-
data
119-
};
120-
} catch (e: any) {
121-
Sentry.captureException(e);
122-
logger.error(`Failed to fetch projects for user ${user}.`, e);
123-
124-
const status = e?.cause?.response?.status;
125-
if (status === 404) {
126-
logger.error(`User ${user} not found or no access`);
147+
logger.debug(`Found ${data.length} projects owned by user ${user} in ${durationMs}ms.`);
127148
return {
128-
type: 'notFound' as const,
129-
value: user
149+
type: 'valid' as const,
150+
data
130151
};
131-
}
132-
throw e;
133-
}
134-
}));
152+
} catch (e: any) {
153+
Sentry.captureException(e);
154+
logger.error(`Failed to fetch projects for user ${user}.`, e);
135155

136-
throwIfAnyFailed(results);
137-
const { validItems: validRepos, notFoundItems: notFoundUsers } = processPromiseResults(results);
156+
const status = e?.cause?.response?.status;
157+
if (status === 404) {
158+
logger.error(`User ${user} not found or no access`);
159+
return {
160+
type: 'notFound' as const,
161+
value: user
162+
};
163+
}
164+
throw e;
165+
}
166+
}));
167+
168+
allResults.push(...batchResults);
169+
}
170+
const { validItems: validRepos, notFoundItems: notFoundUsers } = processPromiseResults(allResults);
138171
allRepos = allRepos.concat(validRepos);
139172
notFound.users = notFoundUsers;
140173
}
@@ -169,7 +202,6 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig, o
169202
}
170203
}));
171204

172-
throwIfAnyFailed(results);
173205
const { validItems: validRepos, notFoundItems: notFoundRepos } = processPromiseResults(results);
174206
allRepos = allRepos.concat(validRepos);
175207
notFound.repos = notFoundRepos;

‎packages/backend/src/repoManager.ts

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -165,7 +165,9 @@ export class RepoManager implements IRepoManager {
165165
});
166166

167167
if (repos.length > 0) {
168-
await this.scheduleRepoIndexingBulk(repos);
168+
for (let i = 0; i < repos.length; i += 100) {
169+
await this.scheduleRepoIndexingBulk(repos.slice(i, i + 100));
170+
}
169171
}
170172
}
171173

‎packages/web/src/actions.ts

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -920,15 +920,17 @@ export const flagConnectionForSync = async (connectionId: number, domain: string
920920
export const flagReposForIndex = async (repoIds: number[], domain: string) => sew(() =>
921921
withAuth((userId) =>
922922
withOrgMembership(userId, domain, async ({ org }) => {
923-
await prisma.repo.updateMany({
924-
where: {
925-
id: { in: repoIds },
926-
orgId: org.id,
927-
},
928-
data: {
929-
repoIndexingStatus: RepoIndexingStatus.NEW,
930-
}
931-
});
923+
for (let i = 0; i < repoIds.length; i += 1000) {
924+
await prisma.repo.updateMany({
925+
where: {
926+
id: { in: repoIds.slice(i, i + 1000) },
927+
orgId: org.id,
928+
},
929+
data: {
930+
repoIndexingStatus: RepoIndexingStatus.NEW,
931+
}
932+
});
933+
}
932934

933935
return {
934936
success: true,
@@ -2152,4 +2154,4 @@ export const encryptValue = async (value: string) => {
21522154

21532155
export const decryptValue = async (iv: string, encryptedValue: string) => {
21542156
return decrypt(iv, encryptedValue);
2155-
}
2157+
}

‎packages/web/src/initialize.ts

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -67,16 +67,18 @@ const syncConnections = async (connections?: { [key: string]: ConnectionConfig }
6767
// Re-try any repos that failed to index.
6868
const failedRepos = currentConnection?.repos.filter(repo => repo.repo.repoIndexingStatus === RepoIndexingStatus.FAILED).map(repo => repo.repo.id) ?? [];
6969
if (failedRepos.length > 0) {
70-
await prisma.repo.updateMany({
71-
where: {
72-
id: {
73-
in: failedRepos,
70+
for (let i = 0; i < failedRepos.length; i += 100) {
71+
await prisma.repo.updateMany({
72+
where: {
73+
id: {
74+
in: failedRepos.slice(i, i + 100),
75+
}
76+
},
77+
data: {
78+
repoIndexingStatus: RepoIndexingStatus.NEW,
7479
}
75-
},
76-
data: {
77-
repoIndexingStatus: RepoIndexingStatus.NEW,
78-
}
79-
})
80+
})
81+
}
8082
}
8183
}
8284
}

0 commit comments

Comments
 (0)