Skip to content

Commit 9b92230

Browse files
author
amvanbaren
committed
Web extension resources return incorrect MIME type
Use [Content_Types].xml to set FileResource contentType Add test case for default content type: application/octet-stream Add migration to set FileResource.contentType Set contentType for RESOURCE type Prepend dot ('.') if extension doesn't start with a dot # Conflicts: # server/src/main/java/org/eclipse/openvsx/ExtensionProcessor.java # server/src/main/java/org/eclipse/openvsx/entities/FileResource.java # server/src/main/java/org/eclipse/openvsx/migration/ExtractResourcesJobRequestHandler.java # server/src/main/java/org/eclipse/openvsx/migration/ExtractResourcesJobService.java # server/src/main/java/org/eclipse/openvsx/migration/MigrationRunner.java # server/src/main/java/org/eclipse/openvsx/migration/MigrationService.java # server/src/main/java/org/eclipse/openvsx/migration/RenameDownloadsJobRequestHandler.java # server/src/main/java/org/eclipse/openvsx/migration/RenameDownloadsService.java # server/src/main/java/org/eclipse/openvsx/migration/SetPreReleaseJobRequestHandler.java # server/src/main/java/org/eclipse/openvsx/migration/SetPreReleaseJobService.java # server/src/main/java/org/eclipse/openvsx/repositories/RepositoryService.java # server/src/main/java/org/eclipse/openvsx/storage/AzureBlobStorageService.java # server/src/main/java/org/eclipse/openvsx/storage/GoogleCloudStorageService.java # server/src/main/java/org/eclipse/openvsx/storage/StorageUtilService.java # server/src/main/jooq-gen/org/eclipse/openvsx/jooq/Indexes.java # server/src/main/jooq-gen/org/eclipse/openvsx/jooq/tables/ExtensionVersion.java # server/src/test/java/org/eclipse/openvsx/adapter/VSCodeAPITest.java # server/src/test/java/org/eclipse/openvsx/repositories/RepositoryServiceSmokeTest.java
1 parent 21bae4f commit 9b92230

21 files changed

+446
-98
lines changed

‎server/src/main/java/org/eclipse/openvsx/ExtensionProcessor.java‎

Lines changed: 56 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -16,24 +16,18 @@
1616
import com.fasterxml.jackson.dataformat.xml.XmlMapper;
1717
import org.apache.commons.codec.digest.DigestUtils;
1818
import org.apache.commons.lang3.StringUtils;
19+
import org.apache.commons.io.FilenameUtils;
1920
import org.eclipse.openvsx.entities.ExtensionVersion;
2021
import org.eclipse.openvsx.entities.FileResource;
2122
import org.eclipse.openvsx.util.*;
2223
import org.slf4j.Logger;
2324
import org.slf4j.LoggerFactory;
2425
import org.springframework.data.util.Pair;
26+
import org.springframework.http.MediaType;
27+
import org.xml.sax.SAXException;
2528

26-
import java.io.EOFException;
27-
import java.io.IOException;
28-
import java.nio.charset.StandardCharsets;
29-
import java.nio.file.Files;
30-
import java.util.*;
31-
import java.util.function.Consumer;
32-
import java.util.function.Function;
33-
import java.util.regex.Pattern;
34-
import java.util.stream.Collectors;
35-
import java.util.zip.ZipException;
36-
import java.util.zip.ZipFile;
29+
import javax.xml.parsers.DocumentBuilderFactory;
30+
import javax.xml.parsers.ParserConfigurationException;
3731

3832
/**
3933
* Processes uploaded extension files and extracts their metadata.
@@ -294,17 +288,22 @@ private List<String> getEngines(JsonNode node) {
294288
}
295289

296290
public List<FileResource> getFileResources(ExtensionVersion extVersion) {
297-
var resources = new ArrayList<FileResource>();
291+
readInputStream();
292+
var contentTypes = loadContentTypes();
298293
var mappers = List.<Function<ExtensionVersion, FileResource>>of(
299294
this::getManifest, this::getReadme, this::getChangelog, this::getLicense, this::getIcon, this::getVsixManifest
300295
);
301296

302-
mappers.forEach(mapper -> Optional.of(extVersion).map(mapper).ifPresent(resources::add));
303-
return resources;
297+
return mappers.stream()
298+
.map(mapper -> mapper.apply(extVersion))
299+
.filter(Objects::nonNull)
300+
.map(resource -> setContentType(resource, contentTypes))
301+
.collect(Collectors.toList());
304302
}
305303

306304
public void processEachResource(ExtensionVersion extVersion, Consumer<FileResource> processor) {
307305
readInputStream();
306+
var contentTypes = loadContentTypes();
308307
zipFile.stream()
309308
.filter(zipEntry -> !zipEntry.isDirectory())
310309
.map(zipEntry -> {
@@ -323,6 +322,7 @@ public void processEachResource(ExtensionVersion extVersion, Consumer<FileResour
323322
resource.setName(zipEntry.getName());
324323
resource.setType(FileResource.RESOURCE);
325324
resource.setContent(bytes);
325+
setContentType(resource, contentTypes);
326326
return resource;
327327
})
328328
.filter(Objects::nonNull)
@@ -417,9 +417,7 @@ public FileResource getLicense(ExtensionVersion extVersion) {
417417
var fileName = matcher.group("file");
418418
var bytes = ArchiveUtil.readEntry(zipFile, "extension/" + fileName);
419419
if (bytes != null) {
420-
var lastSegmentIndex = fileName.lastIndexOf('/');
421-
var lastSegment = fileName.substring(lastSegmentIndex + 1);
422-
license.setName(lastSegment);
420+
license.setName(FilenameUtils.getName(fileName));
423421
license.setContent(bytes);
424422
detectLicense(bytes, extVersion);
425423
return license;
@@ -437,6 +435,44 @@ public FileResource getLicense(ExtensionVersion extVersion) {
437435
return license;
438436
}
439437

438+
private Map<String, String> loadContentTypes() {
439+
var bytes = ArchiveUtil.readEntry(zipFile, "[Content_Types].xml");
440+
var contentTypes = parseContentTypesXml(bytes);
441+
contentTypes.putIfAbsent(".vsix", "application/zip");
442+
return contentTypes;
443+
}
444+
445+
private Map<String, String> parseContentTypesXml(byte[] content) {
446+
var contentTypes = new HashMap<String, String>();
447+
try (var input = new ByteArrayInputStream(content)) {
448+
var document = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(input);
449+
var elements = document.getDocumentElement().getElementsByTagName("Default");
450+
for(var i = 0; i < elements.getLength(); i++) {
451+
var element = elements.item(i);
452+
var attributes = element.getAttributes();
453+
var extension = attributes.getNamedItem("Extension").getTextContent();
454+
if(!extension.startsWith(".")) {
455+
extension = "." + extension;
456+
}
457+
458+
var contentType = attributes.getNamedItem("ContentType").getTextContent();
459+
contentTypes.put(extension, contentType);
460+
}
461+
} catch (IOException | ParserConfigurationException | SAXException e) {
462+
logger.error("failed to read content types", e);
463+
contentTypes.clear();
464+
}
465+
466+
return contentTypes;
467+
}
468+
469+
private FileResource setContentType(FileResource resource, Map<String, String> contentTypes) {
470+
var fileExtension = FilenameUtils.getExtension(resource.getName());
471+
var contentType = contentTypes.getOrDefault(fileExtension, MediaType.APPLICATION_OCTET_STREAM_VALUE);
472+
resource.setContentType(contentType);
473+
return resource;
474+
}
475+
440476
private void detectLicense(byte[] content, ExtensionVersion extVersion) {
441477
if (StringUtils.isEmpty(extVersion.getLicense())) {
442478
var detection = new LicenseDetection();
@@ -449,9 +485,7 @@ private Pair<byte[], String> readFromAlternateNames(String[] names) {
449485
var entry = ArchiveUtil.getEntryIgnoreCase(zipFile, name);
450486
if (entry != null) {
451487
var bytes = ArchiveUtil.readEntry(zipFile, entry);
452-
var lastSegmentIndex = entry.getName().lastIndexOf('/');
453-
var lastSegment = entry.getName().substring(lastSegmentIndex + 1);
454-
return Pair.of(bytes, lastSegment);
488+
return Pair.of(bytes, FilenameUtils.getName(entry.getName()));
455489
}
456490
}
457491
return null;
@@ -466,13 +500,10 @@ protected FileResource getIcon(ExtensionVersion extVersion) {
466500
var bytes = ArchiveUtil.readEntry(zipFile, "extension/" + iconPathStr);
467501
if (bytes == null)
468502
return null;
503+
469504
var icon = new FileResource();
470505
icon.setExtension(extVersion);
471-
var fileNameIndex = iconPathStr.lastIndexOf('/');
472-
if (fileNameIndex >= 0)
473-
icon.setName(iconPathStr.substring(fileNameIndex + 1));
474-
else
475-
icon.setName(iconPathStr);
506+
icon.setName(FilenameUtils.getName(iconPathStr));
476507
icon.setType(FileResource.ICON);
477508
icon.setContent(bytes);
478509
return icon;

‎server/src/main/java/org/eclipse/openvsx/adapter/LocalVSCodeService.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -423,7 +423,7 @@ private ResponseEntity<byte[]> browseFile(
423423
String version
424424
) {
425425
if (resource.getStorageType().equals(FileResource.STORAGE_DB)) {
426-
var headers = storageUtil.getFileResponseHeaders(resource.getName());
426+
var headers = storageUtil.getFileResponseHeaders(resource);
427427
return new ResponseEntity<>(resource.getContent(), headers, HttpStatus.OK);
428428
} else {
429429
var namespace = new Namespace();

‎server/src/main/java/org/eclipse/openvsx/entities/FileResource.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,8 @@ public class FileResource {
4747
@Basic(fetch = FetchType.LAZY)
4848
byte[] content;
4949

50+
String contentType;
51+
5052
@Column(length = 32)
5153
String storageType;
5254

@@ -90,6 +92,14 @@ public void setContent(byte[] content) {
9092
this.content = content;
9193
}
9294

95+
public String getContentType() {
96+
return contentType;
97+
}
98+
99+
public void setContentType(String contentType) {
100+
this.contentType = contentType;
101+
}
102+
93103
public String getStorageType() {
94104
return storageType;
95105
}

‎server/src/main/java/org/eclipse/openvsx/migration/ExtractResourcesJobRequestHandler.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@
1212
import org.eclipse.openvsx.ExtensionProcessor;
1313
import org.eclipse.openvsx.util.NamingUtil;
1414
import org.jobrunr.jobs.annotations.Job;
15-
import org.jobrunr.jobs.context.JobRunrDashboardLogger;
1615
import org.jobrunr.jobs.lambdas.JobRequestHandler;
1716
import org.slf4j.Logger;
1817
import org.slf4j.LoggerFactory;
@@ -24,7 +23,7 @@
2423
@ConditionalOnProperty(value = "ovsx.data.mirror.enabled", havingValue = "false", matchIfMissing = true)
2524
public class ExtractResourcesJobRequestHandler implements JobRequestHandler<MigrationJobRequest> {
2625

27-
protected final Logger logger = new JobRunrDashboardLogger(LoggerFactory.getLogger(ExtractResourcesJobRequestHandler.class));
26+
protected final Logger logger = LoggerFactory.getLogger(ExtractResourcesJobRequestHandler.class);
2827

2928
@Autowired
3029
ExtractResourcesJobService service;
@@ -53,5 +52,6 @@ public void run(MigrationJobRequest jobRequest) throws Exception {
5352
}
5453

5554
service.deleteWebResources(extVersion);
55+
migrations.deleteFile(extensionFile);
5656
}
5757
}

‎server/src/main/java/org/eclipse/openvsx/migration/MigrationRunner.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ public void run(HandlerJobRequest<?> jobRequest) throws Exception {
4646
fixTargetPlatformMigration();
4747
generateSha256ChecksumMigration();
4848
extensionVersionSignatureMigration();
49+
setContentTypeMigration();
4950
}
5051

5152
private void extractResourcesMigration() {
@@ -66,6 +67,12 @@ private void renameDownloadsMigration() {
6667
repositories.findNotMigratedRenamedDownloads().forEach(item -> migrations.enqueueMigration(jobName, handler, item));
6768
}
6869

70+
private void setContentTypeMigration() {
71+
var jobName = "SetContentTypeMigration";
72+
var handler = SetContentTypeJobRequestHandler.class;
73+
repositories.findNotMigratedContentTypes().forEach(item -> enqueueJob(jobName, handler, item));
74+
}
75+
6976
private void extractVsixManifestMigration() {
7077
var jobName = "ExtractVsixManifestMigration";
7178
var handler = ExtractVsixManifestsJobRequestHandler.class;

‎server/src/main/java/org/eclipse/openvsx/migration/MigrationService.java‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
import java.nio.charset.StandardCharsets;
3232
import java.nio.file.Files;
3333
import java.util.AbstractMap;
34+
import java.util.AbstractMap;
3435
import java.util.Map;
3536
import java.util.UUID;
3637

@@ -78,9 +79,8 @@ public TempFile getExtensionFile(Map.Entry<FileResource, byte[]> entry) throws I
7879

7980
var content = entry.getValue();
8081
if(content == null) {
81-
var download = entry.getKey();
82-
var storage = getStorage(download);
83-
var uri = storage.getLocation(download);
82+
var storage = getStorage(resource);
83+
var uri = storage.getLocation(resource);
8484
backgroundRestTemplate.execute("{extensionLocation}", HttpMethod.GET, null, response -> {
8585
try(var out = Files.newOutputStream(extensionFile.getPath())) {
8686
response.getBody().transferTo(out);
@@ -95,12 +95,21 @@ public TempFile getExtensionFile(Map.Entry<FileResource, byte[]> entry) throws I
9595
return extensionFile;
9696
}
9797

98+
public void deleteFile(Path filePath) {
99+
try {
100+
Files.delete(filePath);
101+
} catch (IOException e) {
102+
throw new RuntimeException("Failed to delete file");
103+
}
104+
}
105+
98106
@Retryable
99107
public void uploadFileResource(FileResource resource) {
100108
if(resource.getStorageType().equals(FileResource.STORAGE_DB)) {
101109
return;
102110
}
103111

112+
resource = entityManager.merge(resource);
104113
var storage = getStorage(resource);
105114
storage.uploadFile(resource);
106115
resource.setContent(null);

‎server/src/main/java/org/eclipse/openvsx/migration/SetContentTypeJobRequestHandler.java‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
/** ******************************************************************************
2+
* Copyright (c) 2022 Precies. Software Ltd and others
3+
*
4+
* This program and the accompanying materials are made available under the
5+
* terms of the Eclipse Public License v. 2.0 which is available at
6+
* http://www.eclipse.org/legal/epl-2.0.
7+
*
8+
* SPDX-License-Identifier: EPL-2.0
9+
* ****************************************************************************** */
10+
package org.eclipse.openvsx.migration;
11+
12+
import org.eclipse.openvsx.ExtensionProcessor;
13+
import org.eclipse.openvsx.entities.ExtensionVersion;
14+
import org.eclipse.openvsx.entities.FileResource;
15+
import org.jobrunr.jobs.annotations.Job;
16+
import org.jobrunr.jobs.lambdas.JobRequestHandler;
17+
import org.slf4j.Logger;
18+
import org.slf4j.LoggerFactory;
19+
import org.springframework.beans.factory.annotation.Autowired;
20+
import org.springframework.stereotype.Component;
21+
22+
import java.util.AbstractMap;
23+
import java.util.function.Consumer;
24+
25+
@Component
26+
public class SetContentTypeJobRequestHandler implements JobRequestHandler<MigrationJobRequest> {
27+
28+
protected final Logger logger = LoggerFactory.getLogger(SetContentTypeJobRequestHandler.class);
29+
30+
@Autowired
31+
MigrationService migrations;
32+
33+
@Autowired
34+
SetContentTypeJobService service;
35+
36+
@Override
37+
@Job(name = "Set content type for published file resources", retries = 3)
38+
public void run(MigrationJobRequest jobRequest) throws Exception {
39+
var extVersion = migrations.find(jobRequest, ExtensionVersion.class);
40+
logger.info("Set content type for: {}.{}-{}@{}", extVersion.getExtension().getNamespace().getName(), extVersion.getExtension().getName(), extVersion.getVersion(), extVersion.getTargetPlatform());
41+
42+
var entry = migrations.getDownload(extVersion);
43+
var extensionFile = migrations.getFile(entry);
44+
try (var processor = new ExtensionProcessor(extensionFile)) {
45+
Consumer<FileResource> consumer = resource -> {
46+
var existingResource = service.getExistingResource(extVersion, resource);
47+
if(existingResource == null) {
48+
return;
49+
}
50+
51+
var resourceFile = migrations.getFile(new AbstractMap.SimpleEntry<>(existingResource, resource.getContent()));
52+
migrations.deleteResource(existingResource);
53+
migrations.uploadResource(existingResource, resourceFile);
54+
migrations.deleteFile(resourceFile);
55+
};
56+
57+
processor.getFileResources(extVersion).forEach(consumer);
58+
processor.processEachResource(extVersion, consumer);
59+
}
60+
61+
migrations.deleteFile(extensionFile);
62+
}
63+
}

‎server/src/main/java/org/eclipse/openvsx/migration/SetContentTypeJobService.java‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
/** ******************************************************************************
2+
* Copyright (c) 2022 Precies. Software Ltd and others
3+
*
4+
* This program and the accompanying materials are made available under the
5+
* terms of the Eclipse Public License v. 2.0 which is available at
6+
* http://www.eclipse.org/legal/epl-2.0.
7+
*
8+
* SPDX-License-Identifier: EPL-2.0
9+
* ****************************************************************************** */
10+
package org.eclipse.openvsx.migration;
11+
12+
import org.eclipse.openvsx.entities.ExtensionVersion;
13+
import org.eclipse.openvsx.entities.FileResource;
14+
import org.eclipse.openvsx.repositories.RepositoryService;
15+
import org.springframework.beans.factory.annotation.Autowired;
16+
import org.springframework.stereotype.Component;
17+
18+
import javax.transaction.Transactional;
19+
20+
@Component
21+
public class SetContentTypeJobService {
22+
23+
@Autowired
24+
RepositoryService repositories;
25+
26+
@Transactional
27+
public FileResource getExistingResource(ExtensionVersion extVersion, FileResource resource) {
28+
var existingResource = repositories.findFileByTypeAndName(extVersion, resource.getType(), resource.getName());
29+
if(existingResource != null) {
30+
existingResource.setContentType(resource.getContentType());
31+
}
32+
33+
return existingResource;
34+
}
35+
}

‎server/src/main/java/org/eclipse/openvsx/migration/SetPreReleaseJobRequestHandler.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@
1010
package org.eclipse.openvsx.migration;
1111

1212
import org.jobrunr.jobs.annotations.Job;
13-
import org.jobrunr.jobs.context.JobRunrDashboardLogger;
1413
import org.jobrunr.jobs.lambdas.JobRequestHandler;
1514
import org.slf4j.Logger;
1615
import org.slf4j.LoggerFactory;
@@ -22,7 +21,7 @@
2221
@ConditionalOnProperty(value = "ovsx.data.mirror.enabled", havingValue = "false", matchIfMissing = true)
2322
public class SetPreReleaseJobRequestHandler implements JobRequestHandler<MigrationJobRequest> {
2423

25-
protected final Logger logger = new JobRunrDashboardLogger(LoggerFactory.getLogger(ExtractResourcesJobRequestHandler.class));
24+
protected final Logger logger = LoggerFactory.getLogger(SetPreReleaseJobRequestHandler.class);
2625

2726
@Autowired
2827
MigrationService migrations;
@@ -39,6 +38,7 @@ public void run(MigrationJobRequest jobRequest) throws Exception {
3938
try (var extensionFile = migrations.getExtensionFile(entry)) {
4039
service.updatePreviewAndPreRelease(extVersion, extensionFile);
4140
}
41+
migrations.deleteFile(extensionFile);
4242
}
4343
}
4444
}

0 commit comments

Comments
 (0)