Skip to content

Commit e4b3022

Browse files
author
amvanbaren
committed
Publisher statistics demo
1 parent 2ba9f46 commit e4b3022

File tree

7 files changed

+185
-31
lines changed

7 files changed

+185
-31
lines changed

‎server/src/main/java/org/eclipse/openvsx/repositories/ExtensionJooqRepository.java‎

Lines changed: 38 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111

1212
import org.eclipse.openvsx.entities.Extension;
1313
import org.eclipse.openvsx.entities.Namespace;
14+
import org.eclipse.openvsx.statistics.MembershipDownloadCount;
1415
import org.eclipse.openvsx.util.ExtensionId;
1516
import org.eclipse.openvsx.web.SitemapRow;
1617
import org.jooq.Record;
@@ -24,8 +25,7 @@
2425
import java.util.Map;
2526
import java.util.stream.Collectors;
2627

27-
import static org.eclipse.openvsx.jooq.Tables.EXTENSION;
28-
import static org.eclipse.openvsx.jooq.Tables.NAMESPACE;
28+
import static org.eclipse.openvsx.jooq.Tables.*;
2929

3030
@Component
3131
public class ExtensionJooqRepository {
@@ -270,4 +270,40 @@ public boolean hasExtension(String namespace, String extension) {
270270
.and(EXTENSION.NAME.equalIgnoreCase(extension))
271271
);
272272
}
273+
274+
public List<MembershipDownloadCount> findMembershipDownloads(int offset, int limit) {
275+
return dsl.select(USER_DATA.ID, NAMESPACE.NAME, EXTENSION.NAME, EXTENSION.DOWNLOAD_COUNT)
276+
.from(USER_DATA)
277+
.join(NAMESPACE_MEMBERSHIP).on(NAMESPACE_MEMBERSHIP.USER_DATA.eq(USER_DATA.ID))
278+
.join(NAMESPACE).on(NAMESPACE.ID.eq(NAMESPACE_MEMBERSHIP.NAMESPACE))
279+
.join(EXTENSION).on(EXTENSION.NAMESPACE_ID.eq(NAMESPACE.ID))
280+
.where(USER_DATA.PROVIDER.eq("github"))
281+
.and(EXTENSION.ACTIVE.eq(true))
282+
.orderBy(USER_DATA.ID, NAMESPACE.NAME, EXTENSION.NAME)
283+
.offset(offset)
284+
.limit(limit)
285+
.fetch(row -> new MembershipDownloadCount(
286+
row.get(USER_DATA.ID),
287+
row.get(NAMESPACE.NAME),
288+
row.get(EXTENSION.NAME),
289+
row.get(EXTENSION.DOWNLOAD_COUNT))
290+
);
291+
}
292+
293+
public List<MembershipDownloadCount> findMembershipDownloads(String loginName) {
294+
return dsl.select(USER_DATA.ID, NAMESPACE.NAME, EXTENSION.NAME, EXTENSION.DOWNLOAD_COUNT)
295+
.from(USER_DATA)
296+
.join(NAMESPACE_MEMBERSHIP).on(NAMESPACE_MEMBERSHIP.USER_DATA.eq(USER_DATA.ID))
297+
.join(NAMESPACE).on(NAMESPACE.ID.eq(NAMESPACE_MEMBERSHIP.NAMESPACE))
298+
.join(EXTENSION).on(EXTENSION.NAMESPACE_ID.eq(NAMESPACE.ID))
299+
.where(USER_DATA.PROVIDER.eq("github"))
300+
.and(USER_DATA.LOGIN_NAME.eq(loginName))
301+
.and(EXTENSION.ACTIVE.eq(true))
302+
.fetch(row -> new MembershipDownloadCount(
303+
row.get(USER_DATA.ID),
304+
row.get(NAMESPACE.NAME),
305+
row.get(EXTENSION.NAME),
306+
row.get(EXTENSION.DOWNLOAD_COUNT))
307+
);
308+
}
273309
}

‎server/src/main/java/org/eclipse/openvsx/repositories/PublisherStatisticsRepository.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616

1717
public interface PublisherStatisticsRepository extends Repository<PublisherStatistics, Long> {
1818

19-
PublisherStatistics findByYearAndMonthAndUser(int year, int month, UserData user);
19+
PublisherStatistics findByYearAndMonthAndUserId(int year, int month, long userId);
2020

2121
Streamable<PublisherStatistics> findByUser(UserData user);
2222
}

‎server/src/main/java/org/eclipse/openvsx/repositories/RepositoryService.java‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
import org.eclipse.openvsx.entities.*;
1313
import org.eclipse.openvsx.json.QueryRequest;
1414
import org.eclipse.openvsx.json.VersionTargetPlatformsJson;
15+
import org.eclipse.openvsx.statistics.MembershipDownloadCount;
1516
import org.eclipse.openvsx.util.ExtensionId;
1617
import org.eclipse.openvsx.util.NamingUtil;
1718
import org.eclipse.openvsx.web.SitemapRow;
@@ -671,15 +672,19 @@ public Streamable<Extension> findDeprecatedExtensions(Extension replacement) {
671672
return extensionRepo.findByReplacement(replacement);
672673
}
673674

674-
public PublisherStatistics findPublisherStatisticsByYearAndMonthAndUser(int year, int month, UserData user) {
675-
return publisherStatisticsRepo.findByYearAndMonthAndUser(year, month, user);
675+
public PublisherStatistics findPublisherStatisticsByYearAndMonthAndUserId(int year, int month, long userId) {
676+
return publisherStatisticsRepo.findByYearAndMonthAndUserId(year, month, userId);
676677
}
677678

678679
public Streamable<PublisherStatistics> findPublisherStatisticsByUser(UserData user) {
679680
return publisherStatisticsRepo.findByUser(user);
680681
}
681682

682-
public Streamable<UserData> findUsersByProvider(String provider) {
683-
return userDataRepo.findByProvider(provider);
683+
public List<MembershipDownloadCount> findMembershipDownloads(int offset, int limit) {
684+
return extensionJooqRepo.findMembershipDownloads(offset, limit);
685+
}
686+
687+
public List<MembershipDownloadCount> findMembershipDownloads(String loginName) {
688+
return extensionJooqRepo.findMembershipDownloads(loginName);
684689
}
685690
}

‎server/src/main/java/org/eclipse/openvsx/statistics/MembershipDownloadCount.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
/** ******************************************************************************
2+
* Copyright (c) 2025 Precies. Software OU and others
3+
*
4+
* This program and the accompanying materials are made available under the
5+
* terms of the Eclipse Public License v. 2.0 which is available at
6+
* http://www.eclipse.org/legal/epl-2.0.
7+
*
8+
* SPDX-License-Identifier: EPL-2.0
9+
* ****************************************************************************** */
10+
package org.eclipse.openvsx.statistics;
11+
12+
public record MembershipDownloadCount(long userId, String namespace, String extension, long downloadCount) { }

‎server/src/main/java/org/eclipse/openvsx/statistics/PublisherStatisticsDemo.java‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
/** ******************************************************************************
2+
* Copyright (c) 2025 Precies. Software OU and others
3+
*
4+
* This program and the accompanying materials are made available under the
5+
* terms of the Eclipse Public License v. 2.0 which is available at
6+
* http://www.eclipse.org/legal/epl-2.0.
7+
*
8+
* SPDX-License-Identifier: EPL-2.0
9+
* ****************************************************************************** */
10+
package org.eclipse.openvsx.statistics;
11+
12+
import org.eclipse.openvsx.entities.PublisherStatistics;
13+
import org.eclipse.openvsx.repositories.RepositoryService;
14+
import org.eclipse.openvsx.util.NamingUtil;
15+
import org.eclipse.openvsx.util.TimeUtil;
16+
import org.slf4j.Logger;
17+
import org.slf4j.LoggerFactory;
18+
import org.springframework.boot.context.event.ApplicationStartedEvent;
19+
import org.springframework.context.event.EventListener;
20+
import org.springframework.stereotype.Component;
21+
22+
import java.util.HashMap;
23+
import java.util.Map;
24+
import java.util.Random;
25+
import java.util.stream.Collectors;
26+
27+
@Component
28+
public class PublisherStatisticsDemo {
29+
30+
protected final Logger logger = LoggerFactory.getLogger(PublisherStatisticsDemo.class);
31+
32+
private final RepositoryService repositories;
33+
private final StatisticsService service;
34+
35+
public PublisherStatisticsDemo(RepositoryService repositories, StatisticsService service) {
36+
this.repositories = repositories;
37+
this.service = service;
38+
}
39+
40+
@EventListener
41+
public void applicationStarted(ApplicationStartedEvent event) {
42+
var loginName = "amvanbaren";
43+
var user = repositories.findUserByLoginName("github", loginName);
44+
if(user == null || !repositories.findPublisherStatisticsByUser(user).isEmpty()) {
45+
return;
46+
}
47+
48+
var userDownloads = repositories.findMembershipDownloads(loginName);
49+
userDownloads.forEach(i -> logger.info("{}.{}: {}", i.namespace(), i.extension(), i.downloadCount()));
50+
var totals = userDownloads.stream()
51+
.map((i) -> Map.entry(NamingUtil.toExtensionId(i.namespace(), i.extension()), 0L))
52+
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, (a, b) -> a, HashMap::new));
53+
54+
var random = new Random();
55+
var now = TimeUtil.getCurrentUTC();
56+
for(var i = 6; i > 0; i--) {
57+
var date = now.minusMonths(i);
58+
var downloads = new HashMap<String, Long>();
59+
var totalDownloads = new HashMap<String, Long>();
60+
totals.keySet().forEach((key) -> {
61+
var value = random.nextLong(1000L);
62+
downloads.put(key, value);
63+
var total = totals.get(key);
64+
totalDownloads.put(key, total + value);
65+
totals.put(key, total + value);
66+
});
67+
68+
var statistics = new PublisherStatistics();
69+
statistics.setYear(date.getYear());
70+
statistics.setMonth(date.getMonthValue());
71+
statistics.setUser(user);
72+
statistics.setExtensionDownloads(downloads);
73+
statistics.setExtensionTotalDownloads(totalDownloads);
74+
service.savePublisherStatistics(statistics);
75+
}
76+
}
77+
}

‎server/src/main/java/org/eclipse/openvsx/statistics/PublisherStatisticsJobRequestHandler.java‎

Lines changed: 45 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -9,13 +9,16 @@
99
* ****************************************************************************** */
1010
package org.eclipse.openvsx.statistics;
1111

12+
import jakarta.persistence.EntityManager;
1213
import org.eclipse.openvsx.entities.PublisherStatistics;
14+
import org.eclipse.openvsx.entities.UserData;
1315
import org.eclipse.openvsx.repositories.RepositoryService;
1416
import org.eclipse.openvsx.util.NamingUtil;
1517
import org.jobrunr.jobs.lambdas.JobRequestHandler;
1618
import org.springframework.stereotype.Component;
1719

1820
import java.time.LocalDateTime;
21+
import java.util.ArrayList;
1922
import java.util.Collections;
2023
import java.util.Map;
2124
import java.util.stream.Collectors;
@@ -24,10 +27,12 @@
2427
public class PublisherStatisticsJobRequestHandler implements JobRequestHandler<StatisticsJobRequest> {
2528

2629
private final RepositoryService repositories;
30+
private final EntityManager entityManager;
2731
private final StatisticsService service;
2832

29-
public PublisherStatisticsJobRequestHandler(RepositoryService repositories, StatisticsService service) {
33+
public PublisherStatisticsJobRequestHandler(RepositoryService repositories, EntityManager entityManager, StatisticsService service) {
3034
this.repositories = repositories;
35+
this.entityManager = entityManager;
3136
this.service = service;
3237
}
3338

@@ -37,28 +42,46 @@ public void run(StatisticsJobRequest jobRequest) throws Exception {
3742
var month = jobRequest.getMonth();
3843
var prevDate = LocalDateTime.of(year, month, 1, 0, 0).minusMonths(1);
3944

40-
var users = repositories.findUsersByProvider("github");
41-
for(var user : users) {
42-
var extensions = repositories.findExtensions(user);
43-
var totalDownloads = extensions.stream()
44-
.map(e -> Map.entry(NamingUtil.toExtensionId(e), (long) e.getDownloadCount()))
45-
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
45+
var offset = 0;
46+
var limit = 10000;
47+
var userId = -1L;
48+
var userDownloads = new ArrayList<MembershipDownloadCount>();
49+
var membershipDownloads = Collections.<MembershipDownloadCount>emptyList();
50+
do {
51+
for(var membershipDownload : membershipDownloads) {
52+
if(membershipDownload.userId() != userId) {
53+
if(userId != -1L) {
54+
var totalDownloads = userDownloads.stream()
55+
.map(i -> Map.entry(NamingUtil.toExtensionId(i.namespace(), i.extension()), i.downloadCount()))
56+
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
4657

47-
var prevStatistics = repositories.findPublisherStatisticsByYearAndMonthAndUser(prevDate.getYear(), prevDate.getMonthValue(), user);
48-
var prevTotalDownloads = prevStatistics != null ? prevStatistics.getExtensionTotalDownloads() : Collections.<String, Long>emptyMap();
49-
var downloads = totalDownloads.entrySet().stream()
50-
.map(e -> {
51-
var prevDownloads = prevTotalDownloads.getOrDefault(e.getKey(), 0L);
52-
return Map.entry(e.getKey(), e.getValue() - prevDownloads);
53-
})
54-
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
58+
var prevStatistics = repositories.findPublisherStatisticsByYearAndMonthAndUserId(prevDate.getYear(), prevDate.getMonthValue(), userId);
59+
var prevTotalDownloads = prevStatistics != null ? prevStatistics.getExtensionTotalDownloads() : Collections.<String, Long>emptyMap();
60+
var downloads = totalDownloads.entrySet().stream()
61+
.map(e -> {
62+
var prevDownloads = prevTotalDownloads.getOrDefault(e.getKey(), 0L);
63+
return Map.entry(e.getKey(), e.getValue() - prevDownloads);
64+
})
65+
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
5566

56-
var statistics = new PublisherStatistics();
57-
statistics.setYear(year);
58-
statistics.setMonth(month);
59-
statistics.setExtensionDownloads(downloads);
60-
statistics.setExtensionTotalDownloads(totalDownloads);
61-
service.savePublisherStatistics(statistics);
62-
}
67+
var statistics = new PublisherStatistics();
68+
statistics.setYear(year);
69+
statistics.setMonth(month);
70+
statistics.setUser(entityManager.find(UserData.class, userId));
71+
statistics.setExtensionDownloads(downloads);
72+
statistics.setExtensionTotalDownloads(totalDownloads);
73+
service.savePublisherStatistics(statistics);
74+
}
75+
76+
userId = membershipDownload.userId();
77+
userDownloads.clear();
78+
}
79+
80+
userDownloads.add(membershipDownload);
81+
}
82+
83+
membershipDownloads = repositories.findMembershipDownloads(offset, limit);
84+
offset += limit;
85+
} while(!membershipDownloads.isEmpty());
6386
}
6487
}

‎server/src/test/java/org/eclipse/openvsx/repositories/RepositoryServiceSmokeTest.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -223,9 +223,10 @@ void testExecuteQueries() {
223223
() -> repositories.findNotMigratedLocalNamespaceLogos(page),
224224
() -> repositories.findNotMigratedLocalFileResourceContent(page),
225225
() -> repositories.findNotMigratedFileResourceTypeResource(page),
226-
() -> repositories.findPublisherStatisticsByYearAndMonthAndUser(2025, 1, userData),
226+
() -> repositories.findPublisherStatisticsByYearAndMonthAndUserId(2025, 1, userData.getId()),
227227
() -> repositories.findPublisherStatisticsByUser(userData),
228-
() -> repositories.findUsersByProvider("github")
228+
() -> repositories.findMembershipDownloads("loginName"),
229+
() -> repositories.findMembershipDownloads(0, 1)
229230
);
230231

231232
// check that we did not miss anything

0 commit comments

Comments
 (0)