Skip to content

Commit

Permalink
Upgrade to Koji Build Finder 1.5.0
Browse files Browse the repository at this point in the history
  • Loading branch information
@dwalluck @michalszynkiewicz
dwalluck authored and michalszynkiewicz committed Aug 7, 2020
1 parent bd03215 commit 1925052
Show file tree
Hide file tree
Showing 7 changed files with 190 additions and 168 deletions.
10 changes: 10 additions & 0 deletions .editorconfig
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
root = true

[*]
indent_style = space
indent_size = 4
end_of_line = lf
charset = utf-8
trim_trailing_whitespace = true
insert_final_newline = true
max_line_length = off
1 change: 1 addition & 0 deletions .gitattributes
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* -text
122 changes: 3 additions & 119 deletions pig/src/main/java/org/jboss/pnc/bacon/pig/impl/documents/sharedcontent/BrewSearcher.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,101 +18,27 @@

package org.jboss.pnc.bacon.pig.impl.documents.sharedcontent;

import com.fasterxml.jackson.databind.ObjectMapper;
import com.redhat.red.build.finder.BuildConfig;
import com.redhat.red.build.finder.BuildFinder;
import com.redhat.red.build.finder.DistributionAnalyzer;
import com.redhat.red.build.finder.KojiBuild;
import com.redhat.red.build.finder.KojiClientSession;
import com.redhat.red.build.koji.KojiClientException;
import com.redhat.red.build.koji.model.json.util.KojiObjectMapper;
import com.redhat.red.build.koji.model.xmlrpc.KojiArchiveInfo;
import com.redhat.red.build.koji.model.xmlrpc.KojiTagInfo;
import org.jboss.pnc.bacon.config.Config;
import org.jboss.pnc.bacon.pig.impl.utils.ResourceUtils;
import org.jboss.pnc.bacon.pig.impl.utils.BuildFinderUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Properties;
import java.util.stream.Collectors;

/**
* TODO: 1. move out methods that manipulated on SharedContentReportRow TODO: 2. move to a dedicated package
*
*
* @author Michal Szynkiewicz, [email protected] <br>
* Date: 6/19/17
*/
public class BrewSearcher {
private static final Logger log = LoggerFactory.getLogger(BrewSearcher.class);

private static final String KOJI_BUILD_FINDER_CONFIG_ENV = "KOJI_BUILD_FINDER_CONFIG";

private static final String KOJI_BUILD_FINDER_CONFIG_PROP = "koji.build.finder.config";

private static final String KOJI_BUILD_FINDER_CONFIG_TEMPLATE = "koji-build-finder/config.json";

public static BuildConfig getKojiBuildFinderConfigFromFile(final File file) {
try {
final String json = org.apache.commons.io.FileUtils.readFileToString(file, StandardCharsets.UTF_8);
return getKojiBuildFinderConfigFromJson(json);
} catch (IOException e) {
throw new IllegalStateException(
"Failed read Koji Build Finder configuration from file: " + file.getAbsolutePath(),
e);
}
}

public static BuildConfig getKojiBuildFinderConfigFromFile(final String filename) {
return getKojiBuildFinderConfigFromFile(new File(filename));
}

public static BuildConfig getKojiBuildFinderConfigFromResource(final String resourceName) {
Properties props = new Properties();
props.setProperty("KOJI_URL", Config.instance().getActiveProfile().getPig().getKojiHubUrl());

String json = ResourceUtils.extractToStringWithFiltering(KOJI_BUILD_FINDER_CONFIG_TEMPLATE, props);

return getKojiBuildFinderConfigFromJson(json);
}

public static BuildConfig getKojiBuildFinderConfigFromJson(final String json) {
final ObjectMapper mapper = new KojiObjectMapper();

try {
final BuildConfig config = mapper.readValue(json, BuildConfig.class);
return config;
} catch (IOException e) {
throw new IllegalStateException("Failed read Koji Build Finder configuration from json: " + json, e);
}
}

public static BuildConfig getKojiBuildFinderConfig() {
final String propFilename = System.getProperty(KOJI_BUILD_FINDER_CONFIG_PROP);

if (propFilename != null) {
return getKojiBuildFinderConfigFromFile(propFilename);
}

final String envFilename = System.getenv(KOJI_BUILD_FINDER_CONFIG_ENV);

if (envFilename != null) {
return getKojiBuildFinderConfigFromFile(envFilename);
}

return getKojiBuildFinderConfigFromResource(KOJI_BUILD_FINDER_CONFIG_TEMPLATE);
}

public static void fillBrewData(SharedContentReportRow row) {
log.debug("Asking for {}\n", row.toGapv());
List<KojiBuild> builds = getBuilds(row);
Expand All @@ -137,49 +63,7 @@ private static List<KojiBuild> getBuilds(SharedContentReportRow row) {
}

public static List<KojiBuild> getBuilds(final Path filePath) {
KojiClientSession session;
BuildConfig config;

try {
config = getKojiBuildFinderConfig();
session = new KojiClientSession(config.getKojiHubURL());
} catch (KojiClientException e) {
throw new IllegalStateException("Failed to create Koji session", e);
}

DistributionAnalyzer da = new DistributionAnalyzer(Collections.singletonList(filePath.toFile()), config);
Map<String, Collection<String>> checksumTable;
List<KojiBuild> buildList;

try {
checksumTable = da.checksumFiles().asMap();
BuildFinder bf = new BuildFinder(session, config);
Map<Integer, KojiBuild> builds = bf.findBuilds(checksumTable);
buildList = new ArrayList<>(builds.values());
buildList.sort(Comparator.comparingInt(b -> b.getBuildInfo().getId()));
} catch (IOException e) {
throw new IllegalStateException("Failed to find Koji builds", e);
} finally {
session.close();
}

List<KojiArchiveInfo> archiveInfos = buildList.stream()
.map(KojiBuild::getProjectSourcesTgz)
.filter(Objects::nonNull)
.collect(Collectors.toList());

try {
session = new KojiClientSession(config.getKojiHubURL());
session.enrichArchiveTypeInfo(archiveInfos);
} catch (KojiClientException e) {
throw new IllegalStateException("Failed to enrich Koji builds", e);
} finally {
session.close();
}

List<KojiBuild> ret = buildList.stream().skip(1).collect(Collectors.toList());

return Collections.unmodifiableList(ret);
return BuildFinderUtils.findBuilds(filePath, false);
}

private static void fillBuiltBy(SharedContentReportRow row, KojiBuild build) {
Expand Down
59 changes: 12 additions & 47 deletions pig/src/main/java/org/jboss/pnc/bacon/pig/impl/nvr/NvrListGenerator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,85 +17,50 @@
*/
package org.jboss.pnc.bacon.pig.impl.nvr;

import com.redhat.red.build.finder.BuildConfig;
import com.redhat.red.build.finder.BuildFinder;
import com.redhat.red.build.finder.DistributionAnalyzer;
import com.redhat.red.build.finder.KojiBuild;
import com.redhat.red.build.finder.KojiClientSession;
import com.redhat.red.build.finder.report.NVRReport;
import com.redhat.red.build.finder.report.Report;
import com.redhat.red.build.koji.KojiClientException;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.FilenameUtils;
import org.jboss.pnc.bacon.pig.impl.documents.sharedcontent.BrewSearcher;
import org.jboss.pnc.bacon.pig.impl.utils.BuildFinderUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Map;

public class NvrListGenerator {
public final class NvrListGenerator {
private static final Logger log = LoggerFactory.getLogger(NvrListGenerator.class);

private NvrListGenerator() {

}

public static boolean generateNvrList(String repoZipPath, String targetPath) {
log.info("Generating NVR list for {} and saving result to {}", repoZipPath, targetPath);

BuildConfig config = BrewSearcher.getKojiBuildFinderConfig();
DistributionAnalyzer da = new DistributionAnalyzer(Collections.singletonList(new File(repoZipPath)), config);
Map<String, Collection<String>> checksumTable;
List<KojiBuild> builds = BuildFinderUtils.findBuilds(repoZipPath, true);
File outputDirectory = new File(FilenameUtils.getPath(targetPath));
Report nvrReport = new NVRReport(outputDirectory, builds);

try {
checksumTable = da.checksumFiles().asMap();
nvrReport.outputText();
} catch (IOException e) {
log.error("Failed to get checksums for {}: {}", repoZipPath, e.getMessage());
return false;
}

KojiClientSession session;

try {
session = new KojiClientSession(config.getKojiHubURL());
} catch (KojiClientException e) {
log.error("Failed to create Koji session: {}", e.getMessage());
log.error("Failed to write file {}.txt: {}", nvrReport.getBaseFilename(), e.getMessage(), e);
return false;
}

Map<Integer, KojiBuild> builds;

try {
BuildFinder bf = new BuildFinder(session, config);
builds = bf.findBuilds(checksumTable);
} finally {
session.close();
}

List<KojiBuild> buildList = new ArrayList<>(builds.values());
buildList.sort(Comparator.comparingInt(b -> b.getBuildInfo().getId()));
buildList = Collections.unmodifiableList(buildList);

File outputDirectory = new File(FilenameUtils.getPath(targetPath));
Report nvrReport = new NVRReport(outputDirectory, buildList);
nvrReport.outputText();

File srcFile = new File(outputDirectory, nvrReport.getBaseFilename() + ".txt");
File destFile = new File(targetPath);

try {
FileUtils.moveFile(srcFile, destFile);
} catch (IOException e) {
log.error("Failed to move {} to {}: {}", srcFile, destFile, e.getMessage());
log.error("Failed to move {} to {}: {}", srcFile, destFile, e.getMessage(), e);
return false;
}

return destFile.exists();
}

private NvrListGenerator() {
}
}
Loading

0 comments on commit 1925052

Please sign in to comment.