Skip to content

Commit 72706d1

Browse files
amvanbarenamvanbaren
authored
Merge pull request #457 from amvanbaren/bugfix/release-19286bf
Release 19286bf bugfix
2 parents 711475b + 8a8030c commit 72706d1

File tree

6 files changed

+74
-52
lines changed

6 files changed

+74
-52
lines changed

‎server/src/dev/resources/application.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ spring:
99
username: gitpod
1010
password: gitpod
1111
jpa:
12+
open-in-view: false
1213
properties:
1314
hibernate:
1415
dialect: org.hibernate.dialect.PostgreSQLDialect

‎server/src/main/java/org/eclipse/openvsx/ExtensionProcessor.java‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@
3434
import org.eclipse.openvsx.util.LicenseDetection;
3535
import org.eclipse.openvsx.util.TargetPlatform;
3636
import org.elasticsearch.common.Strings;
37+
import org.slf4j.Logger;
38+
import org.slf4j.LoggerFactory;
3739
import org.springframework.data.util.Pair;
3840
import org.springframework.http.HttpStatus;
3941

@@ -53,6 +55,8 @@ public class ExtensionProcessor implements AutoCloseable {
5355

5456
private static final String WEB_EXTENSION_TAG = "__web_extension";
5557

58+
protected final Logger logger = LoggerFactory.getLogger(ExtensionProcessor.class);
59+
5660
private final InputStream inputStream;
5761
private byte[] content;
5862
private ZipFile zipFile;
@@ -313,7 +317,13 @@ protected List<FileResource> getAllResources(ExtensionVersion extension) {
313317
readInputStream();
314318
return zipFile.stream()
315319
.map(zipEntry -> {
316-
var bytes = ArchiveUtil.readEntry(zipFile, zipEntry);
320+
byte[] bytes;
321+
try {
322+
bytes = ArchiveUtil.readEntry(zipFile, zipEntry);
323+
} catch(ErrorResultException exc) {
324+
logger.warn("Failed to read entry", exc);
325+
bytes = null;
326+
}
317327
if (bytes == null) {
318328
return null;
319329
}

‎server/src/main/java/org/eclipse/openvsx/RegistryApplication.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,10 +15,12 @@
1515
import org.eclipse.openvsx.web.ShallowEtagHeaderFilter;
1616
import org.springframework.boot.SpringApplication;
1717
import org.springframework.boot.autoconfigure.SpringBootApplication;
18+
import org.springframework.boot.availability.ApplicationAvailability;
1819
import org.springframework.boot.web.client.RestTemplateBuilder;
1920
import org.springframework.boot.web.servlet.FilterRegistrationBean;
2021
import org.springframework.cache.annotation.EnableCaching;
2122
import org.springframework.context.annotation.Bean;
23+
import org.springframework.core.Ordered;
2224
import org.springframework.http.converter.ByteArrayHttpMessageConverter;
2325
import org.springframework.http.converter.StringHttpMessageConverter;
2426
import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter;
@@ -27,6 +29,7 @@
2729
import org.springframework.scheduling.annotation.EnableScheduling;
2830
import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler;
2931
import org.springframework.web.client.RestTemplate;
32+
3033
import javax.sql.DataSource;
3134

3235
@SpringBootApplication
@@ -69,7 +72,7 @@ public FilterRegistrationBean<ShallowEtagHeaderFilter> shallowEtagHeaderFilter()
6972
var registrationBean = new FilterRegistrationBean<ShallowEtagHeaderFilter>();
7073
registrationBean.setFilter(new ShallowEtagHeaderFilter());
7174
registrationBean.addUrlPatterns("/api/*");
72-
registrationBean.setOrder(Integer.MAX_VALUE);
75+
registrationBean.setOrder(Ordered.LOWEST_PRECEDENCE);
7376

7477
return registrationBean;
7578
}

‎server/src/main/java/org/eclipse/openvsx/db/migration/V1_23__FileResource_Extract_Resources.java‎

Lines changed: 57 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -24,12 +24,14 @@
2424
import org.springframework.web.client.RestTemplate;
2525

2626
import java.io.ByteArrayInputStream;
27+
import java.io.IOException;
2728
import java.sql.Connection;
2829
import java.sql.ResultSet;
2930
import java.sql.SQLException;
3031
import java.util.ArrayList;
3132
import java.util.List;
3233
import java.util.Map;
34+
import java.util.concurrent.CompletableFuture;
3335
import java.util.stream.Collectors;
3436

3537
@Component
@@ -38,10 +40,10 @@ public class V1_23__FileResource_Extract_Resources extends BaseJavaMigration {
3840
private static final String COL_FR_ID = "fr_id";
3941
private static final String COL_FR_NAME = "fr_name";
4042
private static final String COL_FR_TYPE = "fr_type";
41-
private static final String COL_FR_CONTENT = "fr_content";
4243
private static final String COL_FR_STORAGE_TYPE = "fr_storage_type";
4344
private static final String COL_EV_ID = "ev_id";
4445
private static final String COL_EV_VERSION = "ev_version";
46+
private static final String COL_EV_TARGET_PLATFORM = "ev_target_platform";
4547
private static final String COL_E_NAME = "e_name";
4648
private static final String COL_N_NAME = "n_name";
4749

@@ -57,23 +59,29 @@ public class V1_23__FileResource_Extract_Resources extends BaseJavaMigration {
5759
@Override
5860
public void migrate(Context context) throws Exception {
5961
var connection = context.getConnection();
62+
var uploadFutures = new ArrayList<CompletableFuture<Void>>();
6063
var downloads = getAllDownloads(connection);
61-
var resources = extractResources(downloads);
62-
uploadResources(resources);
63-
deleteResources(connection);
64-
insertResources(resources, connection);
65-
deleteWebResources(connection);
64+
for(var download : downloads) {
65+
var resources = extractResources(download, connection);
66+
uploadFutures.addAll(uploadResources(resources));
67+
deleteType("resource", download.getExtension().getId(), connection);
68+
insertResources(resources, connection);
69+
deleteType("web-resource", download.getExtension().getId(), connection);
70+
}
71+
for(var future : uploadFutures) {
72+
future.join();
73+
}
6674
}
6775

6876
private List<FileResource> getAllDownloads(Connection connection) throws SQLException {
6977
var query = "SELECT " +
7078
"fr.id " + COL_FR_ID + ", " +
7179
"fr.name " + COL_FR_NAME + ", " +
7280
"fr.type " + COL_FR_TYPE + ", " +
73-
"fr.content " + COL_FR_CONTENT + ", " +
7481
"fr.storage_type " + COL_FR_STORAGE_TYPE + ", " +
7582
"ev.id " + COL_EV_ID + ", " +
7683
"ev.version " + COL_EV_VERSION + ", " +
84+
"ev.target_platform " + COL_EV_TARGET_PLATFORM + ", " +
7785
"e.name " + COL_E_NAME + ", " +
7886
"n.name " + COL_N_NAME + " " +
7987
"FROM file_resource fr " +
@@ -105,65 +113,73 @@ private FileResource toFileResource(ResultSet result) throws SQLException {
105113
var extVersion = new ExtensionVersion();
106114
extVersion.setId(result.getLong(COL_EV_ID));
107115
extVersion.setVersion(result.getString(COL_EV_VERSION));
116+
extVersion.setTargetPlatform(result.getString(COL_EV_TARGET_PLATFORM));
108117
extVersion.setExtension(extension);
109118

110119
var resource = new FileResource();
111120
resource.setId(result.getLong(COL_FR_ID));
112121
resource.setName(result.getString(COL_FR_NAME));
113122
resource.setType(result.getString(COL_FR_TYPE));
114-
resource.setContent(result.getBytes(COL_FR_CONTENT));
115123
resource.setStorageType(result.getString(COL_FR_STORAGE_TYPE));
116124
resource.setExtension(extVersion);
117125

118126
return resource;
119127
}
120128

121-
private List<FileResource> extractResources(List<FileResource> downloads) {
129+
private List<FileResource> extractResources(FileResource download, Connection connection) throws SQLException, IOException {
122130
var storages = Map.of(
123131
FileResource.STORAGE_GOOGLE, googleStorage,
124132
FileResource.STORAGE_AZURE, azureStorage
125133
);
126134

127-
var resources = new ArrayList<FileResource>();
128-
for(var download : downloads) {
129-
byte[] content;
130-
if(download.getStorageType().equals(FileResource.STORAGE_DB)) {
131-
content = download.getContent();
132-
} else {
133-
var storage = storages.get(download.getStorageType());
134-
var uri = storage.getLocation(download);
135-
content = restTemplate.getForObject(uri, byte[].class);
136-
}
135+
byte[] content;
136+
if(download.getStorageType().equals(FileResource.STORAGE_DB)) {
137+
content = getContent(download.getId(), connection);
138+
} else {
139+
var storage = storages.get(download.getStorageType());
140+
var uri = storage.getLocation(download);
141+
content = restTemplate.getForObject(uri, byte[].class);
142+
}
137143

138-
try(var processor = new ExtensionProcessor(new ByteArrayInputStream(content))) {
139-
var processedResources = processor.getResources(download.getExtension()).stream()
140-
.filter(resource -> resource.getType().equals(FileResource.RESOURCE))
141-
.map(resource -> {
142-
resource.setStorageType(download.getStorageType());
143-
return resource;
144-
})
145-
.collect(Collectors.toList());
144+
try(var input = new ByteArrayInputStream(content)) {
145+
try(var processor = new ExtensionProcessor(input)) {
146+
var resources = new ArrayList<FileResource>();
147+
var allResources = processor.getResources(download.getExtension());
148+
for(var resource : allResources) {
149+
if(resource.getType().equals(FileResource.RESOURCE)) {
150+
resource.setStorageType(download.getStorageType());
151+
resources.add(resource);
152+
}
153+
}
146154

147-
resources.addAll(processedResources);
155+
return resources;
148156
}
149157
}
158+
}
150159

151-
return resources;
160+
private byte[] getContent(long fileResourceId, Connection connection) throws SQLException {
161+
try(var statement = connection.prepareStatement("SELECT content FROM file_resource WHERE id = ?")) {
162+
statement.setLong(1, fileResourceId);
163+
try(var result = statement.executeQuery()) {
164+
return result.next() ? result.getBytes("content"): null;
165+
}
166+
}
152167
}
153168

154-
private void uploadResources(List<FileResource> resources) {
169+
private List<CompletableFuture<Void>> uploadResources(List<FileResource> resources) {
155170
var storages = Map.of(
156171
FileResource.STORAGE_GOOGLE, googleStorage,
157172
FileResource.STORAGE_AZURE, azureStorage
158173
);
159174

160-
for(var resource : resources) {
161-
if(!resource.getStorageType().equals(FileResource.STORAGE_DB)) {
162-
var storage = storages.get(resource.getStorageType());
163-
storage.uploadFile(resource);
164-
resource.setContent(null);
165-
}
166-
}
175+
return resources.stream()
176+
.filter(resource -> !resource.getStorageType().equals(FileResource.STORAGE_DB))
177+
.map(resource -> CompletableFuture.runAsync(() -> {
178+
var storage = storages.get(resource.getStorageType());
179+
storage.uploadFile(resource);
180+
resource.setContent(null);
181+
}))
182+
.collect(Collectors.toList());
167183
}
168184

169185
private void insertResources(List<FileResource> resources, Connection connection) throws SQLException {
@@ -180,16 +196,11 @@ private void insertResources(List<FileResource> resources, Connection connection
180196
}
181197
}
182198

183-
private void deleteResources(Connection connection) throws SQLException {
184-
var query = "DELETE FROM file_resource WHERE type = 'resource'";
185-
try(var statement = connection.prepareStatement(query)) {
186-
statement.executeUpdate();
187-
}
188-
}
189-
190-
private void deleteWebResources(Connection connection) throws SQLException {
191-
var query = "DELETE FROM file_resource WHERE type = 'web-resource'";
199+
private void deleteType(String type, long extensionId, Connection connection) throws SQLException {
200+
var query = "DELETE FROM file_resource WHERE type = ? AND extension_id = ?";
192201
try(var statement = connection.prepareStatement(query)) {
202+
statement.setString(1, type);
203+
statement.setLong(2, extensionId);
193204
statement.executeUpdate();
194205
}
195206
}

‎server/src/main/java/org/eclipse/openvsx/entities/Extension.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ public class Extension {
4040
@ManyToOne
4141
Namespace namespace;
4242

43-
@OneToMany(mappedBy = "extension")
43+
@OneToMany(fetch = FetchType.EAGER, mappedBy = "extension")
4444
List<ExtensionVersion> versions;
4545

4646
boolean active;

‎server/src/main/java/org/eclipse/openvsx/entities/FileResource.java‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,10 +9,8 @@
99
********************************************************************************/
1010
package org.eclipse.openvsx.entities;
1111

12-
import javax.persistence.Basic;
1312
import javax.persistence.Column;
1413
import javax.persistence.Entity;
15-
import javax.persistence.FetchType;
1614
import javax.persistence.GeneratedValue;
1715
import javax.persistence.Id;
1816
import javax.persistence.OneToOne;
@@ -46,7 +44,6 @@ public class FileResource {
4644
@Column(length = 32)
4745
String type;
4846

49-
@Basic(fetch = FetchType.LAZY)
5047
byte[] content;
5148

5249
@Column(length = 32)

0 commit comments

Comments
 (0)