Skip to content

Commit

Permalink
Skip only empty checksums in PNC (#1313)
Browse files Browse the repository at this point in the history
  • Loading branch information
@dwalluck
dwalluck authored Jan 13, 2025
1 parent 9789dbd commit 91d0105
Show file tree
Hide file tree
Showing 3 changed files with 84 additions and 18 deletions.
44 changes: 32 additions & 12 deletions core/src/main/java/org/jboss/pnc/build/finder/core/BuildFinderUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ public final class BuildFinderUtils {

private final DistributionAnalyzer distributionAnalyzer;

private final Map<ChecksumType, String> emptyDigests;
private final Map<ChecksumType, String> emptyFileDigests;

private final Map<ChecksumType, String> emptyZipDigests;

Expand All @@ -75,14 +75,12 @@ public BuildFinderUtils(BuildConfig config, DistributionAnalyzer distributionAna

loadArchiveExtensions(config, session);

LOGGER.debug("Archive extensions: {}", green(archiveExtensions));

emptyDigests = new EnumMap<>(ChecksumType.class);
emptyFileDigests = new EnumMap<>(ChecksumType.class);
emptyZipDigests = new EnumMap<>(ChecksumType.class);

config.getChecksumTypes()
.forEach(
checksumType -> emptyDigests.put(
checksumType -> emptyFileDigests.put(
checksumType,
Hex.encodeHexString(DigestUtils.getDigest(checksumType.getAlgorithm()).digest())));

Expand All @@ -109,29 +107,46 @@ private static byte[] emptyZipBytes() {
return EMPTY_BYTE_ARRAY;
}

public boolean isEmptyFileDigest(Checksum checksum) {
return checksum.getValue().equals(emptyFileDigests.get(checksum.getType()));
}

public boolean isEmptyZipDigest(Checksum checksum) {
return checksum.getValue().equals(emptyZipDigests.get(checksum.getType()));
}

public boolean shouldSkipChecksum(Checksum checksum, Collection<String> filenames) {
if (checksum.getValue().equals(emptyDigests.get(checksum.getType()))) {
if (isEmptyFileDigest(checksum)) {
if (LOGGER.isWarnEnabled()) {
LOGGER.warn("Skipped empty digest for files: {}", red(String.join(", ", filenames)));
LOGGER.warn("Skipped empty file digest for files: {}", red(String.join(", ", filenames)));
}

return true;
}

if (checksum.getValue().equals(emptyZipDigests.get(checksum.getType()))) {
if (isEmptyZipDigest(checksum)) {
if (LOGGER.isWarnEnabled()) {
LOGGER.warn("Skipped empty zip digest for files: {}", red(String.join(", ", filenames)));
}

return true;
}

return shouldSkipExtension(filenames);
}

private boolean shouldSkipExtension(Collection<String> filenames) {
Collection<String> newArchiveExtensions = new ArrayList<>(archiveExtensions.size() + 1);

newArchiveExtensions.addAll(archiveExtensions);
newArchiveExtensions.add("rpm");

if (filenames.stream().noneMatch(filename -> newArchiveExtensions.stream().anyMatch(filename::endsWith))) {
LOGGER.warn("Skipped due to invalid archive extension for files: {}", red(filenames));
return false;
if (filenames.stream()
.noneMatch(
filename -> newArchiveExtensions.stream()
.anyMatch(extension -> filename.endsWith("." + extension)))) {
LOGGER.warn("Skipped due to invalid archive extension for files: {}", red(String.join(", ", filenames)));
return true;
}

return false;
Expand Down Expand Up @@ -353,7 +368,8 @@ private static List<String> getArchiveExtensionsFromKoji(BuildConfig config, Cli
List<String> archiveTypes = config.getArchiveTypes();
List<String> archiveTypesToCheck;

LOGGER.debug("Archive types: {}", green(archiveTypes));
LOGGER.info("Koji archive types: {}", green(allArchiveTypes));
LOGGER.info("Configured archive types: {}", green(archiveTypes));

if (!archiveTypes.isEmpty()) {
if (LOGGER.isDebugEnabled()) {
Expand Down Expand Up @@ -385,6 +401,10 @@ private static List<String> getArchiveExtensionsFromKoji(BuildConfig config, Cli
.flatMap(List::stream)
.toList();
List<String> extensions = config.getArchiveExtensions();

LOGGER.info("Koji archive extensions: {}", green(allArchiveExtensions));
LOGGER.info("Configured archive extensions: {}", green(extensions));

List<String> extensionsToCheck;

if (!extensions.isEmpty()) {
Expand Down
20 changes: 16 additions & 4 deletions core/src/main/java/org/jboss/pnc/build/finder/core/PncBuildFinder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
package org.jboss.pnc.build.finder.core;

import static org.jboss.pnc.build.finder.core.AnsiUtils.green;
import static org.jboss.pnc.build.finder.core.AnsiUtils.red;
import static org.jboss.pnc.build.finder.core.BuildFinderUtils.BUILD_ID_ZERO;
import static org.jboss.pnc.build.finder.core.BuildFinderUtils.isBuildIdZero;

Expand Down Expand Up @@ -225,8 +226,19 @@ private static ConcurrentHashMap<String, PncBuild> groupArtifactsAsPncBuilds(Ite
*/
private Optional<Artifact> findArtifactInPnc(Checksum checksum, Collection<String> fileNames)
throws RemoteResourceException {
if (buildFinderUtils.shouldSkipChecksum(checksum, fileNames)) {
LOGGER.debug("Skipped checksum {} for fileNames {}", checksum, fileNames);
if (buildFinderUtils.isEmptyFileDigest(checksum)) {
LOGGER.warn(
"Skipped empty file checksum {} for files: {}",
red(checksum),
red(String.join(", ", fileNames)));
return Optional.empty();
}

if (buildFinderUtils.isEmptyZipDigest(checksum)) {
LOGGER.warn(
"Skipped empty zip checksum {} for files: {}",
red(checksum),
red(String.join(", ", fileNames)));
return Optional.empty();
}

Expand Down Expand Up @@ -262,7 +274,7 @@ private static int getArtifactQuality(Object obj) {
case TEMPORARY -> -3;
case DELETED -> -4;
default -> {
LOGGER.warn("Unsupported ArtifactQuality! Got: {}", quality);
LOGGER.warn("Unsupported ArtifactQuality! Got: {}", red(quality));
yield -100;
}
};
Expand Down Expand Up @@ -293,7 +305,7 @@ private KojiBuild convertPncBuildToKojiBuild(PncBuild pncBuild) {

// XXX: Can this ever happen?
if (optionalArtifact.isEmpty()) {
LOGGER.warn("Enhanced artifact with checksum {} has missing artifact", artifact.getChecksum());
LOGGER.warn("Enhanced artifact with checksum {} has missing artifact", red(artifact.getChecksum()));
continue;
}

Expand Down
38 changes: 36 additions & 2 deletions core/src/test/java/org/jboss/pnc/build/finder/core/EmptyBuildsTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import java.util.LinkedHashMap;
import java.util.Map;

import org.apache.commons.io.FilenameUtils;
import org.jboss.pnc.build.finder.koji.KojiBuild;
import org.jboss.pnc.build.finder.koji.KojiClientSession;
import org.junit.jupiter.api.BeforeAll;
Expand Down Expand Up @@ -93,15 +94,48 @@ Map<Checksum, Collection<String>> getChecksumTable() {
}

@Test
void shouldSkipOnEmptyZipFile() throws KojiClientException {
void testSkipOnEmptyZipFile() throws KojiClientException {
// 76cdb2bad9582d23c1f6f4d868218d6c is md5 for empty zip file with size of 22 bytes
String filename = "empty.zip";
Collection<String> filenames = Collections.singletonList("empty.zip");
Collection<String> filenames = Collections.singletonList(filename);
Checksum checksum = new Checksum(ChecksumType.md5, "76cdb2bad9582d23c1f6f4d868218d6c", filename, 22);

try (KojiClientSession session = new KojiClientSession(config.getKojiHubURL())) {
BuildFinderUtils utils = new BuildFinderUtils(config, null, session);
assertThat(utils.isEmptyZipDigest(checksum)).isTrue();
assertThat(utils.shouldSkipChecksum(checksum, filenames)).isTrue();
}
}

@Test
void testSkipExtension() throws KojiClientException {
String filename = "file.foo";
String extension = FilenameUtils.getExtension(filename);

assertThat(config.getArchiveExtensions()).doesNotContain(extension);

Collection<String> filenames = Collections.singletonList(filename);
Checksum checksum = new Checksum(ChecksumType.md5, "b3ba80c13aa555c3eb428dbf62e2c48e", filename, -1L);

try (KojiClientSession session = new KojiClientSession(config.getKojiHubURL())) {
BuildFinderUtils utils = new BuildFinderUtils(config, null, session);
assertThat(utils.shouldSkipChecksum(checksum, filenames)).isTrue();
}
}

@Test
void testNotSkipExtension() throws KojiClientException {
String filename = "file.dll";
String extension = FilenameUtils.getExtension(filename);

assertThat(config.getArchiveExtensions()).contains(extension);

Collection<String> filenames = Collections.singletonList(filename);
Checksum checksum = new Checksum(ChecksumType.md5, "b3ba80c13aa555c3eb428dbf62e2c48e", filename, -1L);

try (KojiClientSession session = new KojiClientSession(config.getKojiHubURL())) {
BuildFinderUtils utils = new BuildFinderUtils(config, null, session);
assertThat(utils.shouldSkipChecksum(checksum, filenames)).isFalse();
}
}
}

0 comments on commit 91d0105

Please sign in to comment.