Skip to content

JENKINS-65970 Ability to ignore indexing errors and continue scanning #471

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

JENKINS-65970 Ability to ignore indexing errors and continue scanning #471

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f853960
Draft of ignore indexing errors trait
sfosterdw Oct 2, 2021
efef46c
Apply spotless
sfosterdw Oct 2, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions ...org/jenkinsci/plugins/github_branch_source/AbstractGitHubSCMHeadProcessErrorStrategy.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
package org.jenkinsci.plugins.github_branch_source;

import jenkins.scm.api.SCMHead;

public abstract class AbstractGitHubSCMHeadProcessErrorStrategy {
public abstract boolean shouldIgnore(SCMHead head, Throwable t);

/** {@inheritDoc} */
public abstract boolean equals(Object o);

/** {@inheritDoc} */
public abstract int hashCode();
}
123 changes: 123 additions & 0 deletions ...java/org/jenkinsci/plugins/github_branch_source/GitHubSCMHeadProcessIgnoreErrorTrait.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
package org.jenkinsci.plugins.github_branch_source;

import edu.umd.cs.findbugs.annotations.NonNull;
import hudson.Extension;
import java.util.Objects;
import jenkins.scm.api.SCMHead;
import jenkins.scm.api.SCMHeadCategory;
import jenkins.scm.api.SCMSource;
import jenkins.scm.api.trait.SCMBuilder;
import jenkins.scm.api.trait.SCMSourceContext;
import jenkins.scm.api.trait.SCMSourceTrait;
import jenkins.scm.api.trait.SCMSourceTraitDescriptor;
import org.kohsuke.stapler.DataBoundConstructor;

public class GitHubSCMHeadProcessIgnoreErrorTrait extends SCMSourceTrait {
private boolean ignoreBranch;
private boolean ignorePR;
private boolean ignoreTag;

@DataBoundConstructor
public GitHubSCMHeadProcessIgnoreErrorTrait(
boolean ignoreBranch, boolean ignorePR, boolean ignoreTag) {
this.ignoreBranch = ignoreBranch;
this.ignorePR = ignorePR;
this.ignoreTag = ignoreTag;
}

public boolean isIgnoreBranch() {
return ignoreBranch;
}

public boolean isIgnorePR() {
return ignorePR;
}

public boolean isIgnoreTag() {
return ignoreTag;
}

@Override
protected void decorateContext(SCMSourceContext<?, ?> context) {
GitHubSCMSourceContext githubContext = (GitHubSCMSourceContext) context;
githubContext.withHeadProcessErrorStrategy(
new GitHubSCMHeadProcessIgnoreErrorStrategy(ignoreBranch, ignorePR, ignoreTag));
}

@Override
public boolean includeCategory(@NonNull SCMHeadCategory category) {
return category.isUncategorized();
}

@Extension
public static class DescriptorImpl extends SCMSourceTraitDescriptor {

@NonNull
@Override
public String getDisplayName() {
return "Ignore Indexing Errors";
}

@Override
public Class<? extends SCMBuilder> getBuilderClass() {
return GitHubSCMBuilder.class;
}

@Override
public Class<? extends SCMSourceContext> getContextClass() {
return GitHubSCMSourceContext.class;
}

@Override
public Class<? extends SCMSource> getSourceClass() {
return GitHubSCMSource.class;
}
}

private static final class GitHubSCMHeadProcessIgnoreErrorStrategy
extends AbstractGitHubSCMHeadProcessErrorStrategy {

private boolean ignoreBranch;
private boolean ignorePR;
private boolean ignoreTag;

public GitHubSCMHeadProcessIgnoreErrorStrategy(
boolean ignoreBranch, boolean ignorePR, boolean ignoreTag) {
this.ignoreBranch = ignoreBranch;
this.ignorePR = ignorePR;
this.ignoreTag = ignoreTag;
}

@Override
public boolean shouldIgnore(SCMHead head, Throwable t) {
if (head instanceof BranchSCMHead) {
return ignoreBranch;
}

if (head instanceof PullRequestSCMHead) {
return ignorePR;
}

if (head instanceof GitHubTagSCMHead) {
return ignoreTag;
}

return true;
}

@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
GitHubSCMHeadProcessIgnoreErrorStrategy that = (GitHubSCMHeadProcessIgnoreErrorStrategy) o;
return ignoreBranch == that.ignoreBranch
&& ignorePR == that.ignorePR
&& ignoreTag == that.ignoreTag;
}

@Override
public int hashCode() {
return Objects.hash(ignoreBranch, ignorePR, ignoreTag);
}
}
}
5 changes: 5 additions & 0 deletions src/main/java/org/jenkinsci/plugins/github_branch_source/GitHubSCMProbe.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,11 @@ public SCMProbeStat stat(@NonNull String path) throws IOException {
}
} catch (FileNotFoundException fnf) {
// means that does not exist and this is handled below this try/catch block.
// } catch (IOException e) {
// Throwable t = e.getCause();
// if (!(t instanceof MismatchedInputException)) {
// throw e;
// }
}
return SCMProbeStat.fromType(SCMFile.Type.NONEXISTENT);
}
Expand Down
124 changes: 82 additions & 42 deletions src/main/java/org/jenkinsci/plugins/github_branch_source/GitHubSCMSource.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1043,6 +1043,7 @@ public GHPermissionType fetch(String username)
&& this.shouldRetrieve(observer, event, BranchSCMHead.class)) {
listener.getLogger().format("%n Checking branches...%n");
int count = 0;
int errorCount = 0;
for (final GHBranch branch : request.getBranches()) {
count++;
String branchName = branch.getName();
Expand All @@ -1053,27 +1054,43 @@ public GHPermissionType fetch(String username)
HyperlinkNote.encodeTo(
resolvedRepositoryUrl + "/tree/" + branchName, branchName));
BranchSCMHead head = new BranchSCMHead(branchName);
if (request.process(
head,
new SCMRevisionImpl(head, branch.getSHA1()),
new SCMSourceRequest.ProbeLambda<BranchSCMHead, SCMRevisionImpl>() {
@NonNull
@Override
public SCMSourceCriteria.Probe create(
@NonNull BranchSCMHead head, @Nullable SCMRevisionImpl revisionInfo)
throws IOException, InterruptedException {
return new GitHubSCMProbe(
apiUri, credentials, ghRepository, head, revisionInfo);
}
},
new CriteriaWitness(listener))) {
listener
.getLogger()
.format("%n %d branches were processed (query completed)%n", count);
break;
try {
if (request.process(
head,
new SCMRevisionImpl(head, branch.getSHA1()),
new SCMSourceRequest.ProbeLambda<BranchSCMHead, SCMRevisionImpl>() {
@NonNull
@Override
public SCMSourceCriteria.Probe create(
@NonNull BranchSCMHead head, @Nullable SCMRevisionImpl revisionInfo)
throws IOException, InterruptedException {
return new GitHubSCMProbe(
apiUri, credentials, ghRepository, head, revisionInfo);
}
},
new CriteriaWitness(listener))) {
listener
.getLogger()
.format("%n %d branches were processed (query completed)%n", count);
break;
}
} catch (IOException e) {
if (request.headProcessErrorStrategies().stream()
.anyMatch(item -> item.shouldIgnore(head, e))) {
listener.getLogger().format("%n Error while processing branch %s%n", branchName);
Functions.printStackTrace(e, listener.getLogger());
errorCount++;
} else {
throw e;
}
}
}
listener.getLogger().format("%n %d branches were processed%n", count);
if (errorCount > 0) {
listener
.getLogger()
.format("%n %d branches encountered errors and were orphaned.%n", errorCount);
}
}
if (request.isFetchPRs()
&& !request.isComplete()
Expand All @@ -1099,25 +1116,36 @@ public SCMSourceCriteria.Probe create(
try {
retrievePullRequest(
apiUri, credentials, ghRepository, pr, strategies, request, listener);
} catch (FileNotFoundException e) {
listener.getLogger().format("%n Error while processing pull request %d%n", number);
Functions.printStackTrace(e, listener.getLogger());
errorCount++;
} catch (IOException e) {
PullRequestSCMHead head = new PullRequestSCMHead(pr, "PR", false);
if (e instanceof FileNotFoundException
|| request.headProcessErrorStrategies().stream()
.anyMatch(item -> item.shouldIgnore(head, e))) {
listener
.getLogger()
.format("%n Error while processing pull request %d%n", number);
Functions.printStackTrace(e, listener.getLogger());
errorCount++;
} else {
throw e;
}
}
count++;
}
listener.getLogger().format("%n %d pull requests were processed%n", count);
if (errorCount > 0) {
listener
.getLogger()
.format("%n %d pull requests encountered errors and were orphaned.%n", count);
.format(
"%n %d pull requests encountered errors and were orphaned.%n", errorCount);
}
}
if (request.isFetchTags()
&& !request.isComplete()
&& this.shouldRetrieve(observer, event, GitHubTagSCMHead.class)) {
listener.getLogger().format("%n Checking tags...%n");
int count = 0;
int errorCount = 0;
for (final GHRef tag : request.getTags()) {
String tagName = tag.getRef();
if (!tagName.startsWith(Constants.R_TAGS)) {
Expand Down Expand Up @@ -1154,27 +1182,40 @@ public SCMSourceCriteria.Probe create(
}
}
GitHubTagSCMHead head = new GitHubTagSCMHead(tagName, tagDate);
if (request.process(
head,
new GitTagSCMRevision(head, sha),
new SCMSourceRequest.ProbeLambda<GitHubTagSCMHead, GitTagSCMRevision>() {
@NonNull
@Override
public SCMSourceCriteria.Probe create(
@NonNull GitHubTagSCMHead head, @Nullable GitTagSCMRevision revisionInfo)
throws IOException, InterruptedException {
return new GitHubSCMProbe(
apiUri, credentials, ghRepository, head, revisionInfo);
}
},
new CriteriaWitness(listener))) {
listener
.getLogger()
.format("%n %d tags were processed (query completed)%n", count);
break;
try {
if (request.process(
head,
new GitTagSCMRevision(head, sha),
new SCMSourceRequest.ProbeLambda<GitHubTagSCMHead, GitTagSCMRevision>() {
@NonNull
@Override
public SCMSourceCriteria.Probe create(
@NonNull GitHubTagSCMHead head, @Nullable GitTagSCMRevision revisionInfo)
throws IOException, InterruptedException {
return new GitHubSCMProbe(
apiUri, credentials, ghRepository, head, revisionInfo);
}
},
new CriteriaWitness(listener))) {
listener
.getLogger()
.format("%n %d tags were processed (query completed)%n", count);
break;
}
} catch (IOException e) {
if (e instanceof FileNotFoundException || true) {
listener.getLogger().format("%n Error while processing tag %s%n", tagName);
Functions.printStackTrace(e, listener.getLogger());
errorCount++;
}
}
}
listener.getLogger().format("%n %d tags were processed%n", count);
if (errorCount > 0) {
listener
.getLogger()
.format("%n %d tags encountered errors and were orphaned.%n", errorCount);
}
}
}
listener.getLogger().format("%nFinished examining %s%n%n", fullName);
Expand Down Expand Up @@ -1258,7 +1299,6 @@ private static void retrievePullRequest(
Connector.release(gitHub);
}
}

if (request.process(
new PullRequestSCMHead(pr, branchName, strategy == ChangeRequestCheckoutStrategy.MERGE),
null,
Expand Down
27 changes: 27 additions & 0 deletions src/main/java/org/jenkinsci/plugins/github_branch_source/GitHubSCMSourceContext.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ public class GitHubSCMSourceContext
* requests.
*/
private boolean wantForkPRs;

/** Set of {@link ChangeRequestCheckoutStrategy} to create for each origin pull request. */
@NonNull
private Set<ChangeRequestCheckoutStrategy> originPRStrategies =
Expand All @@ -75,6 +76,9 @@ public class GitHubSCMSourceContext
*/
private final List<AbstractGitHubNotificationStrategy> notificationStrategies = new ArrayList<>();

private final List<AbstractGitHubSCMHeadProcessErrorStrategy> headProcessErrorStrategies =
new ArrayList<>();

/**
* Constructor.
*
Expand Down Expand Up @@ -172,6 +176,10 @@ public final List<AbstractGitHubNotificationStrategy> notificationStrategies() {
}
return Collections.unmodifiableList(notificationStrategies);
}

public final List<AbstractGitHubSCMHeadProcessErrorStrategy> headProcessErrorStrategies() {
return Collections.unmodifiableList(headProcessErrorStrategies);
}
/**
* Returns {@code true} if notifications should be disabled.
*
Expand Down Expand Up @@ -297,6 +305,25 @@ public final GitHubSCMSourceContext withNotificationStrategy(
return this;
}

public final GitHubSCMSourceContext withHeadProcessErrorStrategies(
List<AbstractGitHubSCMHeadProcessErrorStrategy> strategies) {
headProcessErrorStrategies.clear();
for (AbstractGitHubSCMHeadProcessErrorStrategy strategy : strategies) {
if (!headProcessErrorStrategies.contains(strategy)) {
headProcessErrorStrategies.add(strategy);
}
}
return this;
}

public final GitHubSCMSourceContext withHeadProcessErrorStrategy(
AbstractGitHubSCMHeadProcessErrorStrategy strategy) {
if (!headProcessErrorStrategies.contains(strategy)) {
headProcessErrorStrategies.add(strategy);
}
return this;
}

/**
* Defines the notification mode to use in this context.
*
Expand Down
Loading