Skip to content

Commit

Permalink
Use OPTIONS request before falling back to HEAD request when tryi…
Browse files Browse the repository at this point in the history 
…ng to connect to repositories during normalization. (#4506)

* Use `OPTIONS` request before falling back to `HEAD` request when trying to connect to repositories during normalization.

* dont set `knownToExist`

* use `withUri`

* only apply auth once
  • Loading branch information
@pstreef
pstreef authored Sep 24, 2024
1 parent d2c825c commit 25eb0f6
Show file tree
Hide file tree
Showing 2 changed files with 109 additions and 57 deletions.
145 changes: 89 additions & 56 deletions rewrite-maven/src/main/java/org/openrewrite/maven/internal/MavenPomDownloader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import io.micrometer.core.instrument.Metrics;
import io.micrometer.core.instrument.Timer;
import lombok.Getter;
import lombok.Value;
import org.jspecify.annotations.Nullable;
import org.openrewrite.ExecutionContext;
import org.openrewrite.HttpSenderExecutionContextView;
Expand Down Expand Up @@ -733,8 +734,7 @@ Collection<MavenRepository> distinctNormalizedRepositories(
return null;
}

String originalUrl = repository.getUri();
if ("file".equals(URI.create(originalUrl).getScheme())) {
if ("file".equals(URI.create(repository.getUri()).getScheme())) {
return repository;
}
result = mavenCache.getNormalizedRepository(repository);
Expand All @@ -744,63 +744,13 @@ Collection<MavenRepository> distinctNormalizedRepositories(
ctx.getResolutionListener().repositoryAccessFailed(repository.getUri(), new IllegalArgumentException("Repository " + repository.getUri() + " is not HTTP(S)."));
return null;
}

// Always prefer to use https, fallback to http only if https isn't available
// URLs are case-sensitive after the domain name, so it can be incorrect to lowerCase() a whole URL
// This regex accepts any capitalization of the letters in "http"
String httpsUri = repository.getUri().toLowerCase().startsWith("http:") ?
repository.getUri().replaceFirst("[hH][tT][tT][pP]://", "https://") :
repository.getUri();
if (!httpsUri.endsWith("/")) {
httpsUri += "/";
}

HttpSender.Request.Builder request = applyAuthenticationToRequest(repository, httpSender.head(httpsUri));
MavenRepository normalized = null;
try {
sendRequest(request.build());
normalized = repository.withUri(httpsUri).withKnownToExist(true);
} catch (Throwable t) {
if (t instanceof HttpSenderResponseException) {
HttpSenderResponseException e = (HttpSenderResponseException) t;
// response was returned from the server, but it was not a 200 OK. The server therefore exists.
if (e.isServerReached()) {
normalized = repository.withUri(httpsUri);
}
}
if (normalized == null) {
if (!httpsUri.equals(originalUrl)) {
try {
sendRequest(request.url(originalUrl).build());
normalized = new MavenRepository(
repository.getId(),
originalUrl,
repository.getReleases(),
repository.getSnapshots(),
repository.getUsername(),
repository.getPassword(),
repository.getTimeout());
} catch (HttpSenderResponseException e) {
//Response was returned from the server, but it was not a 200 OK. The server therefore exists.
if (e.isServerReached()) {
normalized = new MavenRepository(
repository.getId(),
originalUrl,
repository.getReleases(),
repository.getSnapshots(),
repository.getUsername(),
repository.getPassword(),
repository.getTimeout());
}
} catch (Throwable e) {
// ok to fall through here and cache a null
}
}
}
if (normalized == null) {
ctx.getResolutionListener().repositoryAccessFailed(repository.getUri(), t);
}
normalized = normalizeRepository(repository);
} catch (Throwable e) {
ctx.getResolutionListener().repositoryAccessFailed(repository.getUri(), e);
}

mavenCache.putNormalizedRepository(repository, normalized);
result = Optional.ofNullable(normalized);
}
Expand All @@ -813,6 +763,89 @@ Collection<MavenRepository> distinctNormalizedRepositories(
return result == null || !result.isPresent() ? null : applyAuthenticationToRepository(result.get());
}

@Nullable
MavenRepository normalizeRepository(MavenRepository repository) throws Throwable {
// Always prefer to use https, fallback to http only if https isn't available
// URLs are case-sensitive after the domain name, so it can be incorrect to lowerCase() a whole URL
// This regex accepts any capitalization of the letters in "http"
String originalUrl = repository.getUri();
String httpsUri = originalUrl.toLowerCase().startsWith("http:") ?
repository.getUri().replaceFirst("[hH][tT][tT][pP]://", "https://") :
repository.getUri();
if (!httpsUri.endsWith("/")) {
httpsUri += "/";
}

HttpSender.Request.Builder request = httpSender.options(httpsUri);
if (repository.getTimeout() != null) {
request = request.withConnectTimeout(repository.getTimeout())
.withReadTimeout(repository.getTimeout());
}

ReachabilityResult reachability = reachable(applyAuthenticationToRequest(repository, request));
if (reachability.isSuccess()) {
return repository.withUri(httpsUri);
}
reachability = reachable(applyAuthenticationToRequest(repository, request.withMethod(HttpSender.Method.HEAD).url(httpsUri)));
if (reachability.isReachable()) {
return repository.withUri(httpsUri);
}
if (!originalUrl.equals(httpsUri)) {
reachability = reachable(applyAuthenticationToRequest(repository, request.withMethod(HttpSender.Method.OPTIONS).url(originalUrl)));
if (reachability.isSuccess()) {
return repository.withUri(originalUrl);
}
reachability = reachable(applyAuthenticationToRequest(repository, request.withMethod(HttpSender.Method.HEAD).url(originalUrl)));
if (reachability.isReachable()) {
return repository.withUri(originalUrl);
}
}
// Won't be null if server is unreachable
throw Objects.requireNonNull(reachability.throwable);
}

@Value
private static class ReachabilityResult {
Reachability reachability;
@Nullable
Throwable throwable;

private static ReachabilityResult success() {
return new ReachabilityResult(Reachability.SUCCESS, null);
}

public boolean isReachable() {
return reachability == Reachability.SUCCESS || reachability == Reachability.ERROR;
}

public boolean isSuccess() {
return reachability == Reachability.SUCCESS;
}
}

enum Reachability {
SUCCESS,
ERROR,
UNREACHABLE;
}

private ReachabilityResult reachable(HttpSender.Request.Builder request) {
try {
sendRequest(request.build());
return ReachabilityResult.success();
} catch (Throwable t) {
if (t instanceof HttpSenderResponseException) {
HttpSenderResponseException e = (HttpSenderResponseException) t;
// response was returned from the server, but it was not a 200 OK. The server therefore exists.
if (e.isServerReached()) {
return new ReachabilityResult(Reachability.ERROR, t);
}
}
return new ReachabilityResult(Reachability.UNREACHABLE, t);
}
}


/**
* Replicates Apache Maven's behavior to attempt anonymous download if repository credentials prove invalid
*/
Expand Down
21 changes: 20 additions & 1 deletion rewrite-maven/src/test/java/org/openrewrite/maven/internal/MavenPomDownloaderTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.CsvSource;
import org.junit.jupiter.params.provider.ValueSource;
import org.openrewrite.*;
import org.openrewrite.ipc.http.HttpSender;
Expand Down Expand Up @@ -767,6 +768,25 @@ void connectTimeout() {
.hasMessageContaining("10.0.0.0");
}

@CsvSource(textBlock = """
https://repo1.maven.org/maven2/, https://repo1.maven.org/maven2/
https://repo1.maven.org/maven2, https://repo1.maven.org/maven2/
http://repo1.maven.org/maven2/, https://repo1.maven.org/maven2/
https://oss.sonatype.org/content/repositories/snapshots/, https://oss.sonatype.org/content/repositories/snapshots/
https://artifactory.moderne.ninja/artifactory/moderne-public/, https://artifactory.moderne.ninja/artifactory/moderne-public/
https://repo.maven.apache.org/maven2/, https://repo.maven.apache.org/maven2/
https://jitpack.io/, https://jitpack.io/
""")
@ParameterizedTest
void normalizeRepository(String originalUrl, String expectedUrl) throws Throwable {
MavenPomDownloader downloader = new MavenPomDownloader(new InMemoryExecutionContext());
MavenRepository repository = new MavenRepository("id", originalUrl, null, null, null, null, null);
MavenRepository normalized = downloader.normalizeRepository(repository);
assertThat(normalized).isNotNull();
assertThat(normalized.getUri()).isEqualTo(expectedUrl);
}

private static GroupArtifactVersion createArtifact(Path repository) throws IOException {
Path target = repository.resolve(Paths.get("org", "openrewrite", "rewrite", "1.0.0"));
Path pom = target.resolve("rewrite-1.0.0.pom");
Expand All @@ -787,5 +807,4 @@ private static GroupArtifactVersion createArtifact(Path repository) throws IOExc
Files.write(jar, "I'm a jar".getBytes()); // empty jars get ignored
return new GroupArtifactVersion("org.openrewrite", "rewrite", "1.0.0");
}

}

0 comments on commit 25eb0f6

Please sign in to comment.